test: bound and canonicalize lifecycle descriptor probes

This commit is contained in:
Joseph Doherty
2026-11-10 12:28:40 -05:00
parent 7b353d7bae
commit 642aefe35a
@@ -99,7 +99,7 @@ public sealed class Sdk2LifecycleSoakTests
var cycle = Stopwatch.StartNew();
await ExerciseAsync(rig, config, ++cycles, evidence, ct);
await EventuallyAsync(() => PublicCounts(rig) == (0, 0, 0), ct);
var sample = Capture(rig, config.CertificateManager, watch.Elapsed, cycles, clientRoot, serverRoot);
var sample = await CaptureAsync(rig, config.CertificateManager, watch.Elapsed, cycles, clientRoot, serverRoot, ct);
sample.Sessions.ShouldBe(0); sample.Subscriptions.ShouldBe(0); sample.Items.ShouldBe(0);
sample.BrowsePoints.ShouldBe(0); sample.HistoryPoints.ShouldBe(0);
sample.LiveDataItems.ShouldBe(0); sample.LiveEventItems.ShouldBe(0); sample.MonitoredNodes.ShouldBe(0);
@@ -144,7 +144,7 @@ public sealed class Sdk2LifecycleSoakTests
application = null;
finalClientCores.Sum(CachedStores).ShouldBe(0);
finalServerCores.Sum(CachedStores).ShouldBe(0);
DescriptorCounts(clientRoot, serverRoot).Pki.ShouldBe(0);
(await DescriptorCountsAsync(ct, clientRoot, serverRoot)).Pki.ShouldBe(0);
await LogAsync(evidence, new { kind = "complete", utc = DateTimeOffset.UtcNow, mode, cycles, restarts, seconds = watch.Elapsed.TotalSeconds,
satisfiesTwoHourDuration = mode == "full" && watch.Elapsed.TotalSeconds >= 7200 });
}
@@ -423,12 +423,12 @@ public sealed class Sdk2LifecycleSoakTests
int Descriptors, int PkiDescriptors, long ManagedBytes, long RssBytes, long AllocatedBytes,
int Gen0, int Gen1, int Gen2);
private static Sample Capture(ServerEventRig rig, object clientManager, TimeSpan elapsed, int cycle,
string clientRoot, string serverRoot)
private static async Task<Sample> CaptureAsync(ServerEventRig rig, object clientManager, TimeSpan elapsed, int cycle,
string clientRoot, string serverRoot, CancellationToken ct)
{
var counts = PublicCounts(rig);
var points = rig.Server.CurrentInstance.SessionManager.GetSessions().Select(ContinuationCounts).ToArray();
var descriptors = DescriptorCounts(clientRoot, serverRoot);
var descriptors = await DescriptorCountsAsync(ct, clientRoot, serverRoot);
var retained = GC.GetTotalMemory(forceFullCollection: true);
using var process = Process.GetCurrentProcess();
process.Refresh();
@@ -440,12 +440,74 @@ public sealed class Sdk2LifecycleSoakTests
GC.CollectionCount(0), GC.CollectionCount(1), GC.CollectionCount(2));
}
private static (int Total, int Pki) DescriptorCounts(params string[] roots)
public static bool DescriptorProbeSupported => OperatingSystem.IsLinux() || OperatingSystem.IsMacOS();
[Fact(SkipUnless = nameof(DescriptorProbeSupported), Skip = "Descriptor probes require Linux procfs or macOS lsof.")]
public async Task Descriptor_probe_observes_an_open_PKI_file_through_a_symlink_alias_and_its_close()
{
var ct = TestContext.Current.CancellationToken;
var temporary = Path.Combine(OperatingSystem.IsMacOS() ? "/tmp" : Path.GetTempPath(), $"otua-pki-probe-{Guid.NewGuid():N}");
var pki = Path.Combine(temporary, "pki");
var sibling = Path.Combine(temporary, "pki-other");
var alias = Path.Combine(temporary, "pki-alias");
Directory.CreateDirectory(pki);
Directory.CreateDirectory(sibling);
try
{
// On macOS the temporary root itself can use /var -> /private/var. The additional
// task-owned alias also checks links whose target contains a linked parent component.
Directory.CreateSymbolicLink(alias, pki);
await using var outside = new FileStream(Path.Combine(sibling, "control"), FileMode.Create,
FileAccess.ReadWrite, FileShare.ReadWrite | FileShare.Delete);
CanonicalPath(alias).ShouldBe(CanonicalPath(pki));
var before = (await DescriptorCountsAsync(ct, pki, alias)).Pki;
before.ShouldBe(0, "a sibling sharing the PKI path prefix is outside the root");
int opened;
await using (var held = new FileStream(Path.Combine(alias, "control"), FileMode.Create,
FileAccess.ReadWrite, FileShare.ReadWrite | FileShare.Delete))
{
opened = (await DescriptorCountsAsync(ct, pki, alias)).Pki;
opened.ShouldBeGreaterThan(0, "the positive control must observe the task-owned file opened through the alias");
}
var closed = (await DescriptorCountsAsync(ct, pki, alias)).Pki;
closed.ShouldBe(0, "closing the positive control must release its descriptor");
TestContext.Current.TestOutputHelper!.WriteLine($"PKI descriptor control: before={before}, opened={opened}, closed={closed}; siblingExcluded=true; aliasResolved=true");
}
finally
{
if (Directory.Exists(alias)) Directory.Delete(alias);
if (Directory.Exists(temporary)) Directory.Delete(temporary, recursive: true);
}
}
[Fact(SkipUnless = nameof(DescriptorProbeSupported), Skip = "The bounded probe control uses a task-owned Unix process.")]
public async Task Descriptor_probe_times_out_and_reaps_a_process_that_keeps_both_streams_open()
{
var ct = TestContext.Current.CancellationToken;
using var process = new Process
{
StartInfo = new ProcessStartInfo("/bin/sh")
{
RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false,
ArgumentList = { "-c", "printf control; printf control >&2; exec /bin/sleep 30" },
},
};
var elapsed = Stopwatch.StartNew();
await Should.ThrowAsync<TimeoutException>(() => ReadDescriptorProbeAsync(process, TimeSpan.FromMilliseconds(250), ct));
elapsed.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(4), "timeout includes stream drains and bounded process cleanup");
process.HasExited.ShouldBeTrue("the task-owned timed-out process must be killed and reaped before returning");
TestContext.Current.TestOutputHelper!.WriteLine($"Descriptor timeout control: deadlineMs=250, elapsedMs={elapsed.Elapsed.TotalMilliseconds:F0}, childExited={process.HasExited}");
}
private static async Task<(int Total, int Pki)> DescriptorCountsAsync(CancellationToken ct, params string[] roots)
{
ct.ThrowIfCancellationRequested();
var canonicalRoots = roots.Select(root => CanonicalPath(root)).Distinct(StringComparer.Ordinal).ToArray();
if (OperatingSystem.IsLinux())
{
var targets = Directory.EnumerateFileSystemEntries("/proc/self/fd").Select(path => new FileInfo(path).LinkTarget).ToArray();
return (targets.Length, targets.Count(path => path is not null && roots.Any(root => path.StartsWith(root, StringComparison.Ordinal))));
var targets = Directory.EnumerateFileSystemEntries("/proc/self/fd")
.Select(path => new FileInfo(path).LinkTarget).ToArray();
return (targets.Length, targets.Count(path => IsWithinRoots(path, canonicalRoots)));
}
if (OperatingSystem.IsMacOS())
{
@@ -453,18 +515,84 @@ public sealed class Sdk2LifecycleSoakTests
{ RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false } };
foreach (var arg in new[] { "-a", "-p", Environment.ProcessId.ToString(CultureInfo.InvariantCulture), "-Fnf" })
process.StartInfo.ArgumentList.Add(arg);
process.Start();
var text = process.StandardOutput.ReadToEnd();
_ = process.StandardError.ReadToEnd();
if (!process.WaitForExit(5000)) { process.Kill(); throw new TimeoutException("Descriptor probe stalled."); }
var text = await ReadDescriptorProbeAsync(process, TimeSpan.FromSeconds(5), ct);
process.ExitCode.ShouldBe(0);
var lines = text.Split('\n');
return (lines.Count(line => line.Length > 1 && line[0] == 'f' && char.IsDigit(line[1])),
lines.Count(line => line.StartsWith('n') && roots.Any(root => line.AsSpan(1).StartsWith(root, StringComparison.Ordinal))));
var total = 0;
var pki = 0;
var descriptor = false;
foreach (var line in text.Split('\n'))
{
if (line.StartsWith('f'))
{
descriptor = line.Length > 1 && char.IsDigit(line[1]);
if (descriptor) total++;
}
else if (descriptor && line.StartsWith('n') && IsWithinRoots(line[1..], canonicalRoots)) pki++;
}
return (total, pki);
}
throw new PlatformNotSupportedException("This soak requires Linux procfs or macOS lsof for actual descriptor evidence.");
}
private static async Task<string> ReadDescriptorProbeAsync(Process process, TimeSpan timeout, CancellationToken ct)
{
using var deadline = CancellationTokenSource.CreateLinkedTokenSource(ct);
deadline.CancelAfter(timeout);
process.Start();
var stdout = process.StandardOutput.ReadToEndAsync(deadline.Token);
var stderr = process.StandardError.ReadToEndAsync(deadline.Token);
var exited = process.WaitForExitAsync(deadline.Token);
try
{
// Waiting for exit alone does not bound reads: both redirected pipes must also finish.
await Task.WhenAll(stdout, stderr, exited).WaitAsync(deadline.Token).ConfigureAwait(false);
return await stdout.ConfigureAwait(false);
}
catch (OperationCanceledException)
{
try { if (!process.HasExited) process.Kill(entireProcessTree: true); }
catch (InvalidOperationException) when (process.HasExited) { /* It exited between the check and kill. */ }
await process.WaitForExitAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(2)).ConfigureAwait(false);
// Observe both cancelled drains; never leave a pipe read running after returning.
try { await Task.WhenAll(stdout, stderr).WaitAsync(TimeSpan.FromSeconds(2)).ConfigureAwait(false); }
catch (OperationCanceledException) { }
if (ct.IsCancellationRequested) throw;
throw new TimeoutException("Descriptor probe exceeded its process and redirected-stream deadline.");
}
}
private static bool IsWithinRoots(string? path, IReadOnlyList<string> roots)
{
if (path is null || !Path.IsPathFullyQualified(path)) return false;
// procfs marks an unlinked but still open file; its containing physical directory still counts.
const string deleted = " (deleted)";
if (path.EndsWith(deleted, StringComparison.Ordinal)) path = path[..^deleted.Length];
var canonical = CanonicalPath(path);
return roots.Any(root => canonical.Equals(root, StringComparison.Ordinal)
|| canonical.StartsWith(Path.EndsInDirectorySeparator(root) ? root : root + Path.DirectorySeparatorChar,
StringComparison.Ordinal));
}
private static string CanonicalPath(string path, int depth = 0)
{
if (depth > 32) throw new IOException("Descriptor path link resolution exceeded its bound.");
var full = Path.GetFullPath(path);
var current = Path.GetPathRoot(full)!;
foreach (var component in full[current.Length..].Split(Path.DirectorySeparatorChar, StringSplitOptions.RemoveEmptyEntries))
{
current = Path.Combine(current, component);
FileSystemInfo entry = Directory.Exists(current) ? new DirectoryInfo(current) : new FileInfo(current);
if (entry.LinkTarget is not null)
{
var target = entry.ResolveLinkTarget(returnFinalTarget: true)
?? throw new IOException("Descriptor path link resolution returned no target.");
// Resolve parent links too: an alias target can itself begin with /var or /tmp.
current = CanonicalPath(target.FullName, depth + 1);
}
}
return Path.TrimEndingDirectorySeparator(current);
}
private static async Task EventuallyAsync(Func<bool> condition, CancellationToken ct)
{
var deadline = Stopwatch.StartNew();