test: implement fixed Linux late-slope diagnostic mode
This commit is contained in:
Joseph Doherty
+206
@@ -0,0 +1,206 @@
|
||||
using System.Diagnostics;
|
||||
using System.Runtime.ExceptionServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Opc.Ua.Configuration;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
|
||||
|
||||
public sealed partial class Sdk2LifecycleSoakTests
|
||||
{
|
||||
private sealed class LateSlopeRun
|
||||
{
|
||||
internal const string Mode = "late-slope";
|
||||
internal const int Duration = 7200;
|
||||
internal const int StartupDeadline = 120;
|
||||
internal const int PhaseDeadline = 7230;
|
||||
internal const int CycleDeadline = 30;
|
||||
|
||||
private readonly List<ExceptionDispatchInfo> _failures = [];
|
||||
private double _shutdownSeconds;
|
||||
private bool _shutdownVerified;
|
||||
|
||||
internal object? ClientManager { get; set; }
|
||||
internal bool PhaseStarted { get; set; }
|
||||
internal bool DurationCompleted { get; set; }
|
||||
internal double LastCompletedCycleSeconds { get; set; }
|
||||
|
||||
internal static void ValidateRequest(int seconds)
|
||||
{
|
||||
seconds.ShouldBe(Duration, "late-slope diagnostic duration is fixed at 7200 seconds");
|
||||
OperatingSystem.IsLinux().ShouldBeTrue("this diagnostic is qualified only for Linux");
|
||||
RuntimeInformation.ProcessArchitecture.ShouldBe(Architecture.Arm64);
|
||||
Environment.Version.ToString().ShouldBe("10.0.14");
|
||||
Environment.GetEnvironmentVariable("OPCUA_CERTIFICATE_LEAK_TRACKING").ShouldBe("1");
|
||||
string.IsNullOrEmpty(Environment.GetEnvironmentVariable("OTOPCUA_SDK2_SATURATION_OBSERVER")).ShouldBeTrue(
|
||||
"this diagnostic does not attach the saturation observer");
|
||||
}
|
||||
|
||||
internal static CancellationTokenSource StartCycleDeadline(CancellationToken token) =>
|
||||
StartLinkedDeadline(token, TimeSpan.FromSeconds(CycleDeadline));
|
||||
|
||||
internal static CancellationTokenSource StartLinkedDeadline(CancellationToken token, TimeSpan duration)
|
||||
{
|
||||
var deadline = CancellationTokenSource.CreateLinkedTokenSource(token);
|
||||
deadline.CancelAfter(duration);
|
||||
return deadline;
|
||||
}
|
||||
|
||||
internal static Task LogStartAsync(StreamWriter evidence, object start, double startupSeconds)
|
||||
{
|
||||
var row = JsonSerializer.SerializeToNode(start)!.AsObject();
|
||||
row["runtime"] = Environment.Version.ToString();
|
||||
row["architecture"] = RuntimeInformation.ProcessArchitecture.ToString();
|
||||
row["startupSeconds"] = startupSeconds;
|
||||
row["acceptance"] = false;
|
||||
return LogAsync(evidence, row);
|
||||
}
|
||||
|
||||
internal void Record(Exception error) => _failures.Add(ExceptionDispatchInfo.Capture(error));
|
||||
|
||||
internal async Task ReleaseOwnersAsync(Func<Task>? server, Func<Task>? client)
|
||||
{
|
||||
// The canceled operation token cannot skip either independently bounded owner.
|
||||
await AttemptAsync(server, "server host");
|
||||
await AttemptAsync(client, "client application");
|
||||
|
||||
async Task AttemptAsync(Func<Task>? dispose, string owner)
|
||||
{
|
||||
if (dispose is null) return;
|
||||
try { await dispose().WaitAsync(TimeSpan.FromSeconds(20), CancellationToken.None); }
|
||||
catch (Exception error) { Record(new InvalidOperationException($"Late-slope {owner} cleanup failed.", error)); }
|
||||
}
|
||||
}
|
||||
|
||||
internal async Task ShutdownAsync(ServerEventRig? rig, ApplicationInstance? application,
|
||||
string clientRoot, string? serverRoot)
|
||||
{
|
||||
var watch = Stopwatch.StartNew();
|
||||
var failuresBeforeShutdown = _failures.Count;
|
||||
object? serverManager = null;
|
||||
object[]? cores = null;
|
||||
if (PhaseStarted)
|
||||
{
|
||||
try
|
||||
{
|
||||
serverManager = rig?.Server.CertificateManager ?? throw new InvalidOperationException("Missing final server manager.");
|
||||
cores = CertificateCores(ClientManager!).Concat(CertificateCores(serverManager)).ToArray();
|
||||
}
|
||||
catch (Exception error) { Record(error); }
|
||||
}
|
||||
|
||||
await ReleaseOwnersAsync(rig is null ? null : () => rig.DisposeAsync().AsTask(),
|
||||
application is null ? null : () => application.DisposeAsync().AsTask());
|
||||
try { if (Directory.Exists(clientRoot)) Directory.Delete(clientRoot, true); }
|
||||
catch (Exception error) { Record(error); }
|
||||
|
||||
if (PhaseStarted && cores is not null && _failures.Count == failuresBeforeShutdown)
|
||||
{
|
||||
try
|
||||
{
|
||||
RequiredField<bool>(ClientManager!, "m_disposed").ShouldBeTrue();
|
||||
RequiredField<bool>(serverManager!, "m_disposed").ShouldBeTrue();
|
||||
cores.Sum(CachedStores).ShouldBe(0);
|
||||
using var probe = new CancellationTokenSource(TimeSpan.FromSeconds(10));
|
||||
(await DescriptorCountsAsync(probe.Token, clientRoot, serverRoot!)).Pki.ShouldBe(0);
|
||||
Directory.Exists(clientRoot).ShouldBeFalse();
|
||||
Directory.Exists(serverRoot!).ShouldBeFalse();
|
||||
_shutdownVerified = true;
|
||||
}
|
||||
catch (Exception error) { Record(error); }
|
||||
}
|
||||
_shutdownSeconds = watch.Elapsed.TotalSeconds;
|
||||
}
|
||||
|
||||
internal async Task EmitAndPreserveFailuresAsync(StreamWriter evidence, int cycles, int restarts,
|
||||
double cycleSeconds, ExceptionDispatchInfo? firstRejectedBudget)
|
||||
{
|
||||
try
|
||||
{
|
||||
await LogAsync(evidence, new { kind = "late-slope-result", mode = Mode, cycles, restarts,
|
||||
cycleSeconds = PhaseStarted ? cycleSeconds : 0, shutdownSeconds = _shutdownSeconds,
|
||||
durationCompleted = DurationCompleted, shutdownVerified = _shutdownVerified,
|
||||
coldBudgetRejected = firstRejectedBudget is not null, fatalFailure = _failures.Count != 0,
|
||||
failureTypes = _failures.Select(x => x.SourceException.GetType().FullName).ToArray(), acceptance = false });
|
||||
}
|
||||
catch (Exception error) { Record(error); }
|
||||
ThrowFailures(firstRejectedBudget);
|
||||
}
|
||||
|
||||
internal void ThrowFailures(ExceptionDispatchInfo? firstRejectedBudget)
|
||||
{
|
||||
if (_failures.Count == 0) { firstRejectedBudget?.Throw(); return; }
|
||||
if (firstRejectedBudget is null && _failures.Count == 1) _failures[0].Throw();
|
||||
var exceptions = _failures.Select(x => x.SourceException).ToList();
|
||||
if (firstRejectedBudget is not null) exceptions.Insert(0, firstRejectedBudget.SourceException);
|
||||
throw new AggregateException("Late-slope diagnostic failed; original resource rejection retained when present.", exceptions);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The cycle deadline cancels the token used by an in-flight asynchronous operation.</summary>
|
||||
[Fact]
|
||||
public async Task Late_slope_cycle_deadline_cancels_in_flight_operation()
|
||||
{
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
using var deadline = LateSlopeRun.StartLinkedDeadline(ct, TimeSpan.FromMilliseconds(75));
|
||||
var operation = Task.Delay(Timeout.InfiniteTimeSpan, deadline.Token);
|
||||
await Should.ThrowAsync<OperationCanceledException>(() => operation.WaitAsync(TimeSpan.FromSeconds(5), ct));
|
||||
operation.IsCanceled.ShouldBeTrue();
|
||||
ct.IsCancellationRequested.ShouldBeFalse();
|
||||
}
|
||||
|
||||
/// <summary>A failing server cleanup cannot skip the client or erase the original budget failure.</summary>
|
||||
[Fact]
|
||||
public async Task Late_slope_cleanup_preserves_first_rejection_and_attempts_both_owners()
|
||||
{
|
||||
var run = new LateSlopeRun { PhaseStarted = true };
|
||||
var original = new ShouldAssertException("first cold RSS rejection");
|
||||
var later = new InvalidOperationException("server cleanup fault");
|
||||
var clientDisposed = false;
|
||||
await run.ReleaseOwnersAsync(() => Task.FromException(later), () =>
|
||||
{
|
||||
clientDisposed = true;
|
||||
return Task.CompletedTask;
|
||||
});
|
||||
clientDisposed.ShouldBeTrue();
|
||||
using var buffer = new MemoryStream();
|
||||
await using var evidence = new StreamWriter(buffer, leaveOpen: true);
|
||||
var error = await Should.ThrowAsync<AggregateException>(() =>
|
||||
run.EmitAndPreserveFailuresAsync(evidence, 4, 1, 16, ExceptionDispatchInfo.Capture(original)));
|
||||
error.InnerExceptions.Count.ShouldBe(2);
|
||||
error.InnerExceptions[0].ShouldBeSameAs(original);
|
||||
error.InnerExceptions[1].InnerException.ShouldBeSameAs(later);
|
||||
using var row = ReadSingleLateSlopeResult(buffer);
|
||||
row.RootElement.GetProperty("fatalFailure").GetBoolean().ShouldBeTrue();
|
||||
row.RootElement.GetProperty("shutdownVerified").GetBoolean().ShouldBeFalse();
|
||||
}
|
||||
|
||||
/// <summary>The real emitter retains the first resource failure and cannot write ordinary completion.</summary>
|
||||
[Fact]
|
||||
public async Task Late_slope_emits_only_diagnostic_result_before_rethrowing_original_budget_failure()
|
||||
{
|
||||
var run = new LateSlopeRun { PhaseStarted = true, DurationCompleted = true };
|
||||
var original = new ShouldAssertException("original cold RSS rejection");
|
||||
using var buffer = new MemoryStream();
|
||||
await using var evidence = new StreamWriter(buffer, leaveOpen: true);
|
||||
var error = await Should.ThrowAsync<ShouldAssertException>(() =>
|
||||
run.EmitAndPreserveFailuresAsync(evidence, 1800, 450, 7201, ExceptionDispatchInfo.Capture(original)));
|
||||
error.ShouldBeSameAs(original);
|
||||
using var row = ReadSingleLateSlopeResult(buffer);
|
||||
row.RootElement.GetProperty("durationCompleted").GetBoolean().ShouldBeTrue();
|
||||
row.RootElement.GetProperty("fatalFailure").GetBoolean().ShouldBeFalse();
|
||||
}
|
||||
|
||||
private static JsonDocument ReadSingleLateSlopeResult(MemoryStream buffer)
|
||||
{
|
||||
var line = Encoding.UTF8.GetString(buffer.ToArray()).Split('\n', StringSplitOptions.RemoveEmptyEntries).ShouldHaveSingleItem();
|
||||
var row = JsonDocument.Parse(line);
|
||||
row.RootElement.GetProperty("kind").GetString().ShouldBe("late-slope-result");
|
||||
row.RootElement.GetProperty("acceptance").GetBoolean().ShouldBeFalse();
|
||||
row.RootElement.GetProperty("coldBudgetRejected").GetBoolean().ShouldBeTrue();
|
||||
return row;
|
||||
}
|
||||
}
|
||||
+159
-88
@@ -44,11 +44,13 @@ public sealed partial class Sdk2LifecycleSoakTests
|
||||
public async Task Repeated_lifecycles_return_resources_to_a_steady_baseline()
|
||||
{
|
||||
var mode = Environment.GetEnvironmentVariable("OTOPCUA_SDK2_SOAK_MODE");
|
||||
(mode is "diagnostic" or "profile" or "full" or "saturation").ShouldBeTrue("an explicit soak mode is required");
|
||||
(mode is "diagnostic" or "profile" or "full" or "saturation" or LateSlopeRun.Mode).ShouldBeTrue("an explicit soak mode is required");
|
||||
var saturation = mode == "saturation" ? SaturationControl.FromEnvironment() : null;
|
||||
var lateSlope = mode == LateSlopeRun.Mode ? new LateSlopeRun() : null;
|
||||
var seconds = int.Parse(Environment.GetEnvironmentVariable("OTOPCUA_SDK2_SOAK_SECONDS")
|
||||
?? (mode == "full" ? "7200" : saturation is not null ? "1800" : "90"), CultureInfo.InvariantCulture);
|
||||
?? (mode == "full" || lateSlope is not null ? "7200" : saturation is not null ? "1800" : "90"), CultureInfo.InvariantCulture);
|
||||
if (mode == "full") seconds.ShouldBeGreaterThanOrEqualTo(7200);
|
||||
else if (lateSlope is not null) LateSlopeRun.ValidateRequest(seconds);
|
||||
else if (saturation is not null) seconds.ShouldBe(1800, "the prospective saturation cutoff is fixed");
|
||||
else if (mode == "profile") seconds.ShouldBeInRange(60, 1800); // Investigative only; never satisfies the full-duration flag.
|
||||
else seconds.ShouldBeInRange(60, 120);
|
||||
@@ -58,7 +60,7 @@ public sealed partial class Sdk2LifecycleSoakTests
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(output)!);
|
||||
await using var evidence = new StreamWriter(new FileStream(output, FileMode.CreateNew, FileAccess.Write, FileShare.Read));
|
||||
using var run = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken);
|
||||
run.CancelAfter(TimeSpan.FromSeconds(saturation is null ? seconds + 120 : 120));
|
||||
run.CancelAfter(TimeSpan.FromSeconds(saturation is null && lateSlope is null ? seconds + 120 : LateSlopeRun.StartupDeadline));
|
||||
var ct = run.Token;
|
||||
var watch = Stopwatch.StartNew();
|
||||
var clientRoot = Path.Combine(Path.GetTempPath(), $"otua-soak-client-{Guid.NewGuid():N}");
|
||||
@@ -89,9 +91,21 @@ public sealed partial class Sdk2LifecycleSoakTests
|
||||
serverRoot = RequiredField<string>(rig, "_pkiRoot");
|
||||
Configure(rig);
|
||||
// The required observation period excludes initial certificate creation and server startup.
|
||||
var startupSeconds = watch.Elapsed.TotalSeconds;
|
||||
if (lateSlope is not null)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
startupSeconds.ShouldBeLessThanOrEqualTo(LateSlopeRun.StartupDeadline);
|
||||
}
|
||||
watch.Restart();
|
||||
if (saturation is not null) run.CancelAfter(TimeSpan.FromSeconds(1800));
|
||||
await LogAsync(evidence, new
|
||||
if (lateSlope is not null)
|
||||
{
|
||||
lateSlope.ClientManager = config.CertificateManager;
|
||||
lateSlope.PhaseStarted = true;
|
||||
run.CancelAfter(TimeSpan.FromSeconds(LateSlopeRun.PhaseDeadline));
|
||||
}
|
||||
var start = new
|
||||
{
|
||||
kind = "start", utc = DateTimeOffset.UtcNow, processId = Environment.ProcessId,
|
||||
mode, requestedSeconds = seconds, minimumFullSeconds = 7200,
|
||||
@@ -100,8 +114,12 @@ public sealed partial class Sdk2LifecycleSoakTests
|
||||
diagnosticInstrumentation = "SDK2.0.1 CP lists and certificate-validation cached stores; read-only",
|
||||
limits = new { retainedManagedGrowthBytes = 64L << 20, rssGrowthBytes = 128L << 20, fdGrowth = 32 },
|
||||
coverage = "encrypted sessions; scalar/Int16[] Raw+UNS; subscription removal; browse abandonment; history release/cross-session/close; surgical changes/rebuild; revocation; paused TCP read/recovery; same-PKI restart",
|
||||
});
|
||||
while (saturation is not null ? cycles < 385 : watch.Elapsed < TimeSpan.FromSeconds(seconds))
|
||||
};
|
||||
if (lateSlope is null) await LogAsync(evidence, start);
|
||||
else await LateSlopeRun.LogStartAsync(evidence, start, startupSeconds);
|
||||
while (saturation is not null ? cycles < 385
|
||||
: lateSlope is not null ? lateSlope.LastCompletedCycleSeconds < LateSlopeRun.Duration
|
||||
: watch.Elapsed < TimeSpan.FromSeconds(seconds))
|
||||
{
|
||||
if (saturation is not null)
|
||||
{
|
||||
@@ -115,101 +133,144 @@ public sealed partial class Sdk2LifecycleSoakTests
|
||||
}
|
||||
}
|
||||
var cycle = Stopwatch.StartNew();
|
||||
await ExerciseAsync(rig, config, ++cycles, evidence, ct);
|
||||
await EventuallyAsync(() => PublicCounts(rig) == (0, 0, 0), ct);
|
||||
var sample = await CaptureAsync(rig, config.CertificateManager, watch.Elapsed, cycles, clientRoot, serverRoot, ct);
|
||||
// Retain the failing observation too; rejected diagnostic rows never count as passing quiescent samples.
|
||||
await LogAsync(evidence, new { kind = "resource-observation", resources = sample });
|
||||
sample.Sessions.ShouldBe(0); sample.Subscriptions.ShouldBe(0); sample.Items.ShouldBe(0);
|
||||
sample.BrowsePoints.ShouldBe(0); sample.HistoryPoints.ShouldBe(0);
|
||||
sample.LiveDataItems.ShouldBe(0); sample.LiveEventItems.ShouldBe(0); sample.MonitoredNodes.ShouldBe(0);
|
||||
sample.PkiDescriptors.ShouldBe(0, "quiescent certificate file descriptors must be closed");
|
||||
if (cycles >= 3)
|
||||
using var cycleDeadline = lateSlope is null ? null : LateSlopeRun.StartCycleDeadline(ct);
|
||||
var cycleToken = cycleDeadline?.Token ?? ct;
|
||||
try
|
||||
{
|
||||
baseline ??= sample;
|
||||
sample.CachedCertificateStores.ShouldBe(baseline.CachedCertificateStores);
|
||||
sample.Variables.ShouldBe(baseline.Variables); sample.Folders.ShouldBe(baseline.Folders);
|
||||
sample.Conditions.ShouldBe(baseline.Conditions);
|
||||
if (firstRejectedBudget is null)
|
||||
await ExerciseAsync(rig, config, ++cycles, evidence, cycleToken);
|
||||
await EventuallyAsync(() => PublicCounts(rig) == (0, 0, 0), cycleToken);
|
||||
var sample = await CaptureAsync(rig, config.CertificateManager, watch.Elapsed, cycles, clientRoot, serverRoot, cycleToken);
|
||||
if (lateSlope is not null) sample = sample with { Seconds = watch.Elapsed.TotalSeconds };
|
||||
// Retain the failing observation too; rejected diagnostic rows never count as passing quiescent samples.
|
||||
await LogAsync(evidence, new { kind = "resource-observation", resources = sample });
|
||||
sample.Sessions.ShouldBe(0); sample.Subscriptions.ShouldBe(0); sample.Items.ShouldBe(0);
|
||||
sample.BrowsePoints.ShouldBe(0); sample.HistoryPoints.ShouldBe(0);
|
||||
sample.LiveDataItems.ShouldBe(0); sample.LiveEventItems.ShouldBe(0); sample.MonitoredNodes.ShouldBe(0);
|
||||
sample.PkiDescriptors.ShouldBe(0, "quiescent certificate file descriptors must be closed");
|
||||
if (cycles >= 3)
|
||||
{
|
||||
try
|
||||
baseline ??= sample;
|
||||
sample.CachedCertificateStores.ShouldBe(baseline.CachedCertificateStores);
|
||||
sample.Variables.ShouldBe(baseline.Variables); sample.Folders.ShouldBe(baseline.Folders);
|
||||
sample.Conditions.ShouldBe(baseline.Conditions);
|
||||
if (firstRejectedBudget is null)
|
||||
{
|
||||
sample.Descriptors.ShouldBeLessThanOrEqualTo(baseline.Descriptors + 32);
|
||||
sample.ManagedBytes.ShouldBeLessThanOrEqualTo(baseline.ManagedBytes + (64L << 20));
|
||||
sample.RssBytes.ShouldBeLessThanOrEqualTo(baseline.RssBytes + (128L << 20));
|
||||
try
|
||||
{
|
||||
sample.Descriptors.ShouldBeLessThanOrEqualTo(baseline.Descriptors + 32);
|
||||
sample.ManagedBytes.ShouldBeLessThanOrEqualTo(baseline.ManagedBytes + (64L << 20));
|
||||
sample.RssBytes.ShouldBeLessThanOrEqualTo(baseline.RssBytes + (128L << 20));
|
||||
}
|
||||
catch (ShouldAssertException error) when (mode == "profile" || saturation is not null || lateSlope is not null)
|
||||
{
|
||||
// Investigative sampling may continue after rejection, but can never turn it into a pass.
|
||||
firstRejectedBudget = ExceptionDispatchInfo.Capture(error);
|
||||
await LogAsync(evidence, new { kind = "budget-rejected", cycle = cycles,
|
||||
seconds = watch.Elapsed.TotalSeconds, resources = sample, baseline, failure = error.ToString() });
|
||||
}
|
||||
}
|
||||
catch (ShouldAssertException error) when (mode == "profile" || saturation is not null)
|
||||
retained.Enqueue(sample);
|
||||
if (retained.Count > 24) retained.Dequeue();
|
||||
if (retained.Count == 24)
|
||||
{
|
||||
// Investigative sampling may continue after rejection, but can never turn it into a pass.
|
||||
firstRejectedBudget = ExceptionDispatchInfo.Capture(error);
|
||||
await LogAsync(evidence, new { kind = "budget-rejected", cycle = cycles,
|
||||
seconds = watch.Elapsed.TotalSeconds, resources = sample, baseline, failure = error.ToString() });
|
||||
var window = retained.ToArray();
|
||||
var strictlyIncreasing = window.Zip(window.Skip(1)).All(pair => pair.Second.ManagedBytes > pair.First.ManagedBytes);
|
||||
(strictlyIncreasing && window[^1].ManagedBytes - window[0].ManagedBytes > (16L << 20)).ShouldBeFalse(
|
||||
"retained managed memory must not grow monotonically across 24 quiescent cycles");
|
||||
}
|
||||
}
|
||||
retained.Enqueue(sample);
|
||||
if (retained.Count > 24) retained.Dequeue();
|
||||
if (retained.Count == 24)
|
||||
await LogAsync(evidence, new { kind = firstRejectedBudget is null ? "quiescent" : "quiescent-budget-rejected",
|
||||
cycleSeconds = cycle.Elapsed.TotalSeconds, resources = sample });
|
||||
if (cycles % 4 == 0)
|
||||
{
|
||||
var window = retained.ToArray();
|
||||
var strictlyIncreasing = window.Zip(window.Skip(1)).All(pair => pair.Second.ManagedBytes > pair.First.ManagedBytes);
|
||||
(strictlyIncreasing && window[^1].ManagedBytes - window[0].ManagedBytes > (16L << 20)).ShouldBeFalse(
|
||||
"retained managed memory must not grow monotonically across 24 quiescent cycles");
|
||||
var restart = Stopwatch.StartNew();
|
||||
if (lateSlope is null)
|
||||
await RestartWithActiveResourcesAsync(rig, config, cycleToken).WaitAsync(TimeSpan.FromSeconds(25), cycleToken);
|
||||
else
|
||||
{
|
||||
using var restartDeadline = LateSlopeRun.StartLinkedDeadline(cycleToken, TimeSpan.FromSeconds(25));
|
||||
try
|
||||
{
|
||||
await RestartWithActiveResourcesAsync(rig, config, restartDeadline.Token);
|
||||
restartDeadline.Token.ThrowIfCancellationRequested();
|
||||
}
|
||||
finally { restartDeadline.Cancel(); }
|
||||
}
|
||||
Configure(rig);
|
||||
restarts++;
|
||||
await LogAsync(evidence, new { kind = "restart", cycles, restarts, seconds = restart.Elapsed.TotalSeconds });
|
||||
}
|
||||
if (saturation is not null)
|
||||
{
|
||||
if (cycles == 352) saturation.Boundary("window1-end", cycles, restarts);
|
||||
if (cycles == 384) saturation.Boundary("window2-end", cycles, restarts);
|
||||
}
|
||||
if (lateSlope is not null)
|
||||
{
|
||||
cycleToken.ThrowIfCancellationRequested();
|
||||
var cycleSeconds = cycle.Elapsed.TotalSeconds;
|
||||
cycleSeconds.ShouldBeLessThanOrEqualTo(LateSlopeRun.CycleDeadline);
|
||||
var elapsed = watch.Elapsed.TotalSeconds;
|
||||
await LogAsync(evidence, new { kind = "late-slope-cycle", cycle = cycles, restarts,
|
||||
elapsedSeconds = elapsed, cycleSeconds });
|
||||
cycleToken.ThrowIfCancellationRequested();
|
||||
cycle.Elapsed.TotalSeconds.ShouldBeLessThanOrEqualTo(LateSlopeRun.CycleDeadline);
|
||||
lateSlope.LastCompletedCycleSeconds = elapsed;
|
||||
}
|
||||
}
|
||||
await LogAsync(evidence, new { kind = firstRejectedBudget is null ? "quiescent" : "quiescent-budget-rejected",
|
||||
cycleSeconds = cycle.Elapsed.TotalSeconds, resources = sample });
|
||||
if (cycles % 4 == 0)
|
||||
{
|
||||
var restart = Stopwatch.StartNew();
|
||||
await RestartWithActiveResourcesAsync(rig, config, ct).WaitAsync(TimeSpan.FromSeconds(25), ct);
|
||||
Configure(rig);
|
||||
restarts++;
|
||||
await LogAsync(evidence, new { kind = "restart", cycles, restarts, seconds = restart.Elapsed.TotalSeconds });
|
||||
}
|
||||
if (saturation is not null)
|
||||
{
|
||||
if (cycles == 352) saturation.Boundary("window1-end", cycles, restarts);
|
||||
if (cycles == 384) saturation.Boundary("window2-end", cycles, restarts);
|
||||
}
|
||||
finally { cycleDeadline?.Cancel(); }
|
||||
}
|
||||
// Preserve the first assertion and its original stack even if later RSS samples decrease.
|
||||
// This happens before any completion record; finally still releases the owned rig and client.
|
||||
if (saturation is null) firstRejectedBudget?.Throw();
|
||||
var phaseSeconds = watch.Elapsed.TotalSeconds;
|
||||
if (saturation is not null)
|
||||
if (lateSlope is not null)
|
||||
{
|
||||
cycles.ShouldBe(385); restarts.ShouldBe(96);
|
||||
phaseSeconds.ShouldBeLessThanOrEqualTo(1800, "there is no saturation cutoff extension");
|
||||
watch.Stop();
|
||||
ct.ThrowIfCancellationRequested();
|
||||
watch.Elapsed.TotalSeconds.ShouldBeInRange(LateSlopeRun.Duration, LateSlopeRun.PhaseDeadline);
|
||||
cycles.ShouldBeGreaterThanOrEqualTo(4);
|
||||
restarts.ShouldBe(cycles / 4);
|
||||
lateSlope.DurationCompleted = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
cycles.ShouldBeGreaterThanOrEqualTo(4, "diagnostic mode must exercise at least one actual restart");
|
||||
restarts.ShouldBeGreaterThan(0);
|
||||
watch.Elapsed.TotalSeconds.ShouldBeGreaterThanOrEqualTo(seconds);
|
||||
// Preserve the first assertion and its original stack even if later RSS samples decrease.
|
||||
// This happens before any completion record; finally still releases the owned rig and client.
|
||||
if (saturation is null) firstRejectedBudget?.Throw();
|
||||
var phaseSeconds = watch.Elapsed.TotalSeconds;
|
||||
if (saturation is not null)
|
||||
{
|
||||
cycles.ShouldBe(385); restarts.ShouldBe(96);
|
||||
phaseSeconds.ShouldBeLessThanOrEqualTo(1800, "there is no saturation cutoff extension");
|
||||
}
|
||||
else
|
||||
{
|
||||
cycles.ShouldBeGreaterThanOrEqualTo(4, "diagnostic mode must exercise at least one actual restart");
|
||||
restarts.ShouldBeGreaterThan(0);
|
||||
watch.Elapsed.TotalSeconds.ShouldBeGreaterThanOrEqualTo(seconds);
|
||||
}
|
||||
var cleanupToken = saturation is null ? ct : TestContext.Current.CancellationToken;
|
||||
var finalClientCores = CertificateCores(config.CertificateManager);
|
||||
var finalServerCores = CertificateCores(rig.Server.CertificateManager!);
|
||||
await rig.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(20), cleanupToken);
|
||||
rig = null;
|
||||
await application.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(20), cleanupToken);
|
||||
application = null;
|
||||
finalClientCores.Sum(CachedStores).ShouldBe(0);
|
||||
finalServerCores.Sum(CachedStores).ShouldBe(0);
|
||||
(await DescriptorCountsAsync(cleanupToken, clientRoot, serverRoot)).Pki.ShouldBe(0);
|
||||
if (saturation is not null)
|
||||
{
|
||||
await LogAsync(evidence, new { kind = "saturation-result", utc = DateTimeOffset.UtcNow,
|
||||
canonicalCycles = 384, settlingCycles = 1, restarts, phaseSeconds,
|
||||
readiness = saturation.Readiness, coldBudgetRejected = firstRejectedBudget is not null,
|
||||
shutdownOwnershipVerified = true, acceptance = false });
|
||||
firstRejectedBudget?.Throw();
|
||||
saturation.Readiness.ShouldBeTrue("diagnostic readiness requires both fixed observation windows");
|
||||
return;
|
||||
}
|
||||
await LogAsync(evidence, new { kind = "complete", utc = DateTimeOffset.UtcNow, mode, cycles, restarts, seconds = watch.Elapsed.TotalSeconds,
|
||||
satisfiesTwoHourDuration = mode == "full" && watch.Elapsed.TotalSeconds >= 7200 });
|
||||
}
|
||||
var cleanupToken = saturation is null ? ct : TestContext.Current.CancellationToken;
|
||||
var finalClientCores = CertificateCores(config.CertificateManager);
|
||||
var finalServerCores = CertificateCores(rig.Server.CertificateManager!);
|
||||
await rig.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(20), cleanupToken);
|
||||
rig = null;
|
||||
await application.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(20), cleanupToken);
|
||||
application = null;
|
||||
finalClientCores.Sum(CachedStores).ShouldBe(0);
|
||||
finalServerCores.Sum(CachedStores).ShouldBe(0);
|
||||
(await DescriptorCountsAsync(cleanupToken, clientRoot, serverRoot)).Pki.ShouldBe(0);
|
||||
if (saturation is not null)
|
||||
{
|
||||
await LogAsync(evidence, new { kind = "saturation-result", utc = DateTimeOffset.UtcNow,
|
||||
canonicalCycles = 384, settlingCycles = 1, restarts, phaseSeconds,
|
||||
readiness = saturation.Readiness, coldBudgetRejected = firstRejectedBudget is not null,
|
||||
shutdownOwnershipVerified = true, acceptance = false });
|
||||
firstRejectedBudget?.Throw();
|
||||
saturation.Readiness.ShouldBeTrue("diagnostic readiness requires both fixed observation windows");
|
||||
return;
|
||||
}
|
||||
await LogAsync(evidence, new { kind = "complete", utc = DateTimeOffset.UtcNow, mode, cycles, restarts, seconds = watch.Elapsed.TotalSeconds,
|
||||
satisfiesTwoHourDuration = mode == "full" && watch.Elapsed.TotalSeconds >= 7200 });
|
||||
}
|
||||
catch (Exception error) when (lateSlope is not null) { lateSlope.Record(error); }
|
||||
catch (Exception error)
|
||||
{
|
||||
await LogAsync(evidence, new { kind = "failed", cycles, restarts, seconds = watch.Elapsed.TotalSeconds,
|
||||
@@ -219,11 +280,21 @@ public sealed partial class Sdk2LifecycleSoakTests
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (rig is not null) await rig.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(20), TestContext.Current.CancellationToken);
|
||||
if (application is not null) await application.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(20), TestContext.Current.CancellationToken);
|
||||
if (Directory.Exists(clientRoot)) Directory.Delete(clientRoot, true);
|
||||
if (serverRoot is not null) Directory.Exists(serverRoot).ShouldBeFalse("the task-owned server PKI must be released");
|
||||
if (lateSlope is not null)
|
||||
{
|
||||
watch.Stop();
|
||||
await lateSlope.ShutdownAsync(rig, application, clientRoot, serverRoot);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (rig is not null) await rig.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(20), TestContext.Current.CancellationToken);
|
||||
if (application is not null) await application.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(20), TestContext.Current.CancellationToken);
|
||||
if (Directory.Exists(clientRoot)) Directory.Delete(clientRoot, true);
|
||||
if (serverRoot is not null) Directory.Exists(serverRoot).ShouldBeFalse("the task-owned server PKI must be released");
|
||||
}
|
||||
}
|
||||
if (lateSlope is not null)
|
||||
await lateSlope.EmitAndPreserveFailuresAsync(evidence, cycles, restarts, watch.Elapsed.TotalSeconds, firstRejectedBudget);
|
||||
}
|
||||
|
||||
private static void Configure(ServerEventRig rig)
|
||||
|
||||
Reference in New Issue
Block a user