dohertj2/lmxopcua
1
0
Fork 0
Code Issues 38 Pull Requests Actions Packages Projects Releases Wiki Activity

test(opcua): add fixed candidate saturation diagnosis with observer barriers

Browse Source
This commit is contained in:
Joseph Doherty
2026-11-10 14:51:19 -05:00
parent ab35ddd3c2
commit 54a4ff2149
8 changed files with 833 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2LifecycleSoakTests.cs
+269 -13
View File
@@ -1,6 +1,7 @@
using System.Collections;
using System.Collections.Concurrent;
using System.Diagnostics;
using System.Diagnostics.Tracing;
using System.Globalization;
using System.Net;
using System.Net.Sockets;
@@ -39,14 +40,16 @@ public sealed class Sdk2LifecycleSoakTests
private static readonly DateTime Epoch = new(2026, 10, 1, 0, 0, 0, DateTimeKind.Utc);
public static bool Enabled => Environment.GetEnvironmentVariable("OTOPCUA_SDK2_SOAK") == "1";
[Fact(SkipUnless = nameof(Enabled), Skip = "Opt in with OTOPCUA_SDK2_SOAK=1 and an explicit diagnostic/full mode.")]
[Fact(SkipUnless = nameof(Enabled), Skip = "Opt in with OTOPCUA_SDK2_SOAK=1 and an explicit soak mode.")]
public async Task Repeated_lifecycles_return_resources_to_a_steady_baseline()
{
var mode = Environment.GetEnvironmentVariable("OTOPCUA_SDK2_SOAK_MODE");
(mode is "diagnostic" or "profile" or "full").ShouldBeTrue("explicit diagnostic, profile or full mode is required");
(mode is "diagnostic" or "profile" or "full" or "saturation").ShouldBeTrue("an explicit soak mode is required");
var saturation = mode == "saturation" ? SaturationControl.FromEnvironment() : null;
var seconds = int.Parse(Environment.GetEnvironmentVariable("OTOPCUA_SDK2_SOAK_SECONDS")
?? (mode == "full" ? "7200" : "90"), CultureInfo.InvariantCulture);
?? (mode == "full" ? "7200" : saturation is not null ? "1800" : "90"), CultureInfo.InvariantCulture);
if (mode == "full") seconds.ShouldBeGreaterThanOrEqualTo(7200);
else if (saturation is not null) seconds.ShouldBe(1800, "the prospective saturation cutoff is fixed");
else if (mode == "profile") seconds.ShouldBeInRange(60, 1800); // Investigative only; never satisfies the full-duration flag.
else seconds.ShouldBeInRange(60, 120);
var output = Environment.GetEnvironmentVariable("OTOPCUA_SDK2_SOAK_OUTPUT")
@@ -55,7 +58,7 @@ public sealed class Sdk2LifecycleSoakTests
Directory.CreateDirectory(Path.GetDirectoryName(output)!);
await using var evidence = new StreamWriter(new FileStream(output, FileMode.CreateNew, FileAccess.Write, FileShare.Read));
using var run = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken);
run.CancelAfter(TimeSpan.FromSeconds(seconds + 120));
run.CancelAfter(TimeSpan.FromSeconds(saturation is null ? seconds + 120 : 120));
var ct = run.Token;
var watch = Stopwatch.StartNew();
var clientRoot = Path.Combine(Path.GetTempPath(), $"otua-soak-client-{Guid.NewGuid():N}");
@@ -87,6 +90,7 @@ public sealed class Sdk2LifecycleSoakTests
Configure(rig);
// The required observation period excludes initial certificate creation and server startup.
watch.Restart();
if (saturation is not null) run.CancelAfter(TimeSpan.FromSeconds(1800));
await LogAsync(evidence, new
{
kind = "start", utc = DateTimeOffset.UtcNow, processId = Environment.ProcessId,
@@ -97,8 +101,19 @@ public sealed class Sdk2LifecycleSoakTests
limits = new { retainedManagedGrowthBytes = 64L << 20, rssGrowthBytes = 128L << 20, fdGrowth = 32 },
coverage = "encrypted sessions; scalar/Int16[] Raw+UNS; subscription removal; browse abandonment; history release/cross-session/close; surgical changes/rebuild; revocation; paused TCP read/recovery; same-PKI restart",
});
while (watch.Elapsed < TimeSpan.FromSeconds(seconds))
while (saturation is not null ? cycles < 385 : watch.Elapsed < TimeSpan.FromSeconds(seconds))
{
if (saturation is not null)
{
if (cycles == 320) await saturation.AttachAsync(ct);
if (cycles == 320) saturation.Boundary("window1-start", 321, restarts);
if (cycles == 352) saturation.Boundary("window2-start", 353, restarts);
if (cycles == 384)
{
await saturation.DetachAndDrainAsync(ct);
await LogAsync(evidence, new { kind = "saturation-settling-start", cycle = 385, restarts });
}
}
var cycle = Stopwatch.StartNew();
await ExerciseAsync(rig, config, ++cycles, evidence, ct);
await EventuallyAsync(() => PublicCounts(rig) == (0, 0, 0), ct);
@@ -123,7 +138,7 @@ public sealed class Sdk2LifecycleSoakTests
sample.ManagedBytes.ShouldBeLessThanOrEqualTo(baseline.ManagedBytes + (64L << 20));
sample.RssBytes.ShouldBeLessThanOrEqualTo(baseline.RssBytes + (128L << 20));
}
catch (ShouldAssertException error) when (mode == "profile")
catch (ShouldAssertException error) when (mode == "profile" || saturation is not null)
{
// Investigative sampling may continue after rejection, but can never turn it into a pass.
firstRejectedBudget = ExceptionDispatchInfo.Capture(error);
@@ -151,22 +166,47 @@ public sealed class Sdk2LifecycleSoakTests
restarts++;
await LogAsync(evidence, new { kind = "restart", cycles, restarts, seconds = restart.Elapsed.TotalSeconds });
}
if (saturation is not null)
{
if (cycles == 352) saturation.Boundary("window1-end", cycles, restarts);
if (cycles == 384) saturation.Boundary("window2-end", cycles, restarts);
}
}
// Preserve the first assertion and its original stack even if later RSS samples decrease.
// This happens before any completion record; finally still releases the owned rig and client.
firstRejectedBudget?.Throw();
cycles.ShouldBeGreaterThanOrEqualTo(4, "diagnostic mode must exercise at least one actual restart");
restarts.ShouldBeGreaterThan(0);
watch.Elapsed.TotalSeconds.ShouldBeGreaterThanOrEqualTo(seconds);
if (saturation is null) firstRejectedBudget?.Throw();
var phaseSeconds = watch.Elapsed.TotalSeconds;
if (saturation is not null)
{
cycles.ShouldBe(385); restarts.ShouldBe(96);
phaseSeconds.ShouldBeLessThanOrEqualTo(1800, "there is no saturation cutoff extension");
}
else
{
cycles.ShouldBeGreaterThanOrEqualTo(4, "diagnostic mode must exercise at least one actual restart");
restarts.ShouldBeGreaterThan(0);
watch.Elapsed.TotalSeconds.ShouldBeGreaterThanOrEqualTo(seconds);
}
var cleanupToken = saturation is null ? ct : TestContext.Current.CancellationToken;
var finalClientCores = CertificateCores(config.CertificateManager);
var finalServerCores = CertificateCores(rig.Server.CertificateManager!);
await rig.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(20), ct);
await rig.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(20), cleanupToken);
rig = null;
await application.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(20), ct);
await application.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(20), cleanupToken);
application = null;
finalClientCores.Sum(CachedStores).ShouldBe(0);
finalServerCores.Sum(CachedStores).ShouldBe(0);
(await DescriptorCountsAsync(ct, clientRoot, serverRoot)).Pki.ShouldBe(0);
(await DescriptorCountsAsync(cleanupToken, clientRoot, serverRoot)).Pki.ShouldBe(0);
if (saturation is not null)
{
await LogAsync(evidence, new { kind = "saturation-result", utc = DateTimeOffset.UtcNow,
canonicalCycles = 384, settlingCycles = 1, restarts, phaseSeconds,
readiness = saturation.Readiness, coldBudgetRejected = firstRejectedBudget is not null,
shutdownOwnershipVerified = true, acceptance = false });
firstRejectedBudget?.Throw();
saturation.Readiness.ShouldBeTrue("diagnostic readiness requires both fixed observation windows");
return;
}
await LogAsync(evidence, new { kind = "complete", utc = DateTimeOffset.UtcNow, mode, cycles, restarts, seconds = watch.Elapsed.TotalSeconds,
satisfiesTwoHourDuration = mode == "full" && watch.Elapsed.TotalSeconds >= 7200 });
}
@@ -174,6 +214,7 @@ public sealed class Sdk2LifecycleSoakTests
{
await LogAsync(evidence, new { kind = "failed", cycles, restarts, seconds = watch.Elapsed.TotalSeconds,
exceptionType = error.GetType().FullName });
if (saturation is not null) firstRejectedBudget?.Throw();
throw;
}
finally
@@ -631,6 +672,221 @@ public sealed class Sdk2LifecycleSoakTests
await output.WriteLineAsync(JsonSerializer.Serialize(value));
await output.FlushAsync();
}
// Test-only IPC. Receipts are fresh, run/PID-bound and published atomically by a separate observer.
private sealed class SaturationControl(string directory, string runId)
{
private const string Protocol = "sdk2-saturation-v1";
private int _observerPid;
private int _nextMarker;
internal bool Readiness { get; private set; }
internal static SaturationControl FromEnvironment()
{
var directory = Environment.GetEnvironmentVariable("OTOPCUA_SDK2_SATURATION_CONTROL")
?? throw new InvalidOperationException("A fresh saturation control directory is required.");
var runId = Environment.GetEnvironmentVariable("OTOPCUA_SDK2_SATURATION_RUN_ID")
?? throw new InvalidOperationException("An explicit saturation run id is required.");
if (!Path.IsPathFullyQualified(directory) || !Guid.TryParseExact(runId, "D", out _))
throw new InvalidOperationException("Saturation control identity is malformed.");
Directory.CreateDirectory(directory);
if (Directory.EnumerateFileSystemEntries(directory).Any())
throw new InvalidOperationException("Saturation control directory must be empty.");
_ = SaturationMarkers.Log;
return new SaturationControl(directory, runId);
}
internal async Task AttachAsync(CancellationToken ct)
{
await PublishAsync("attach-request.json", new { protocol = Protocol, runId,
targetPid = Environment.ProcessId, firstCycle = 321, completedRestarts = 80 }, ct);
using var receipt = await WaitAsync("active.json", ct);
var root = receipt.RootElement;
ValidateIdentity(root);
root.GetProperty("active").GetBoolean().ShouldBeTrue();
_observerPid = root.GetProperty("observerPid").GetInt32();
if (_observerPid <= 0 || _observerPid == Environment.ProcessId)
throw new InvalidDataException("A distinct observer process is required.");
using var enabled = CancellationTokenSource.CreateLinkedTokenSource(ct);
enabled.CancelAfter(TimeSpan.FromSeconds(5));
while (!SaturationMarkers.Log.IsEnabled()) await Task.Delay(10, enabled.Token);
}
internal void Boundary(string name, int cycle, int restarts)
{
(string Name, int Cycle, int Restarts)[] expected =
[ ("window1-start", 321, 80), ("window1-end", 352, 88),
("window2-start", 353, 88), ("window2-end", 384, 96) ];
if (_observerPid == 0 || _nextMarker >= expected.Length || expected[_nextMarker] != (name, cycle, restarts))
throw new InvalidOperationException("Saturation boundary is out of order.");
if (!SaturationMarkers.Log.IsEnabled()) throw new InvalidOperationException("Saturation observer is not active.");
SaturationMarkers.Log.Boundary(runId, name, cycle, restarts);
_nextMarker++;
}
internal async Task DetachAndDrainAsync(CancellationToken ct)
{
if (_nextMarker != 4) throw new InvalidOperationException("Both fixed observation windows are required.");
await PublishAsync("detach-request.json", new { protocol = Protocol, runId,
targetPid = Environment.ProcessId, lastCycle = 384, completedRestarts = 96 }, ct);
using var receipt = await WaitAsync("drained.json", ct);
Readiness = ValidateDrained(receipt.RootElement, runId, Environment.ProcessId, _observerPid);
using var disabled = CancellationTokenSource.CreateLinkedTokenSource(ct);
disabled.CancelAfter(TimeSpan.FromSeconds(5));
while (SaturationMarkers.Log.IsEnabled()) await Task.Delay(10, disabled.Token);
}
internal static bool ValidateDrained(JsonElement root, string expectedRun, int target, int observer)
{
if (root.GetProperty("protocol").GetString() != Protocol || root.GetProperty("runId").GetString() != expectedRun
|| root.GetProperty("targetPid").GetInt32() != target || root.GetProperty("observerPid").GetInt32() != observer
|| !root.GetProperty("drained").GetBoolean() || root.GetProperty("active").GetBoolean())
throw new InvalidDataException("Saturation drained receipt identity or ownership is invalid.");
var windows = root.GetProperty("windows").EnumerateArray().ToArray();
if (windows.Length != 2) throw new InvalidDataException("Exactly two fixed windows are required.");
var ready = root.GetProperty("markersValid").GetBoolean();
foreach (var name in new[] { "eventsLost", "sdkFactoryJit", "sdkFactoryLoads", "sdkAssemblyLoads" })
{
var value = root.GetProperty(name).GetInt64();
if (value < 0) throw new InvalidDataException("Negative diagnostic count.");
ready &= value == 0;
}
for (var index = 0; index < 2; index++)
{
var window = windows[index];
if (window.GetProperty("name").GetString() != $"window{index + 1}"
|| window.GetProperty("firstCycle").GetInt32() != 321 + index * 32
|| window.GetProperty("lastCycle").GetInt32() != 352 + index * 32
|| window.GetProperty("startRestarts").GetInt32() != 80 + index * 8
|| window.GetProperty("endRestarts").GetInt32() != 88 + index * 8)
throw new InvalidDataException("Saturation window boundaries are invalid.");
foreach (var name in new[] { "sdkFactoryJit", "sdkFactoryLoads", "sdkAssemblyLoads" })
{
var value = window.GetProperty(name).GetInt64();
if (value < 0) throw new InvalidDataException("Negative window diagnostic count.");
ready &= value == 0;
}
}
return ready;
}
private void ValidateIdentity(JsonElement root)
{
if (root.GetProperty("protocol").GetString() != Protocol || root.GetProperty("runId").GetString() != runId
|| root.GetProperty("targetPid").GetInt32() != Environment.ProcessId)
throw new InvalidDataException("Saturation observer receipt identity is invalid.");
}
private async Task PublishAsync(string name, object value, CancellationToken ct)
{
var temporary = Path.Combine(directory, name + ".tmp");
await using (var stream = new FileStream(temporary, FileMode.CreateNew, FileAccess.Write))
await JsonSerializer.SerializeAsync(stream, value, cancellationToken: ct);
File.Move(temporary, Path.Combine(directory, name));
}
private async Task<JsonDocument> WaitAsync(string name, CancellationToken ct)
{
using var bounded = CancellationTokenSource.CreateLinkedTokenSource(ct);
bounded.CancelAfter(TimeSpan.FromSeconds(45));
var path = Path.Combine(directory, name);
while (!File.Exists(path))
{
if (File.Exists(Path.Combine(directory, "observer-failed.json")))
throw new InvalidOperationException("The saturation observer failed before completing its receipt.");
await Task.Delay(25, bounded.Token);
}
return JsonDocument.Parse(await File.ReadAllTextAsync(path, bounded.Token));
}
}
[Fact]
public void Saturation_protocol_rejects_stale_identity_and_nonzero_readiness_counts()
{
var run = Guid.NewGuid().ToString("D");
JsonDocument Receipt(string identity, int pid = 100, int lost = 0, int jit = 0, int loads = 0, int assemblies = 0) =>
JsonDocument.Parse(JsonSerializer.Serialize(new { protocol = "sdk2-saturation-v1", runId = identity,
targetPid = pid, observerPid = 200, active = false, drained = true, markersValid = true,
eventsLost = lost, sdkFactoryJit = jit, sdkFactoryLoads = loads, sdkAssemblyLoads = assemblies,
windows = new[] {
new { name = "window1", firstCycle = 321, lastCycle = 352, startRestarts = 80, endRestarts = 88,
sdkFactoryJit = jit, sdkFactoryLoads = loads, sdkAssemblyLoads = assemblies },
new { name = "window2", firstCycle = 353, lastCycle = 384, startRestarts = 88, endRestarts = 96,
sdkFactoryJit = 0, sdkFactoryLoads = 0, sdkAssemblyLoads = 0 } } }));
using var good = Receipt(run);
SaturationControl.ValidateDrained(good.RootElement, run, 100, 200).ShouldBeTrue();
using var stale = Receipt(Guid.NewGuid().ToString("D"));
Should.Throw<InvalidDataException>(() => SaturationControl.ValidateDrained(stale.RootElement, run, 100, 200));
using var wrongPid = Receipt(run, pid: 101);
Should.Throw<InvalidDataException>(() => SaturationControl.ValidateDrained(wrongPid.RootElement, run, 100, 200));
foreach (var denied in new[] { Receipt(run, lost: 1), Receipt(run, jit: 1), Receipt(run, loads: 1), Receipt(run, assemblies: 1) })
{
using (denied) SaturationControl.ValidateDrained(denied.RootElement, run, 100, 200).ShouldBeFalse();
}
}
[Fact]
public async Task Saturation_protocol_missing_observer_is_bounded()
{
var directory = Path.Combine(Path.GetTempPath(), $"otua-saturation-control-{Guid.NewGuid():N}");
Directory.CreateDirectory(directory);
try
{
using var deadline = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken);
deadline.CancelAfter(TimeSpan.FromMilliseconds(100));
var control = new SaturationControl(directory, Guid.NewGuid().ToString("D"));
await Should.ThrowAsync<OperationCanceledException>(() => control.AttachAsync(deadline.Token));
File.Exists(Path.Combine(directory, "active.json")).ShouldBeFalse();
}
finally { Directory.Delete(directory, recursive: true); }
}
public static bool SaturationObserverAvailable => File.Exists(Environment.GetEnvironmentVariable("OTOPCUA_SDK2_SATURATION_OBSERVER"));
[Theory(SkipUnless = nameof(SaturationObserverAvailable), Skip = "Requires the separately built diagnostic observer.")]
[InlineData(false)]
[InlineData(true)]
public async Task Saturation_protocol_actual_observer_drains_and_rejects_extra_foreign_markers(bool foreignMarker)
{
var directory = Path.Combine(Path.GetTempPath(), $"otua-saturation-observer-{Guid.NewGuid():N}");
Directory.CreateDirectory(directory);
var run = Guid.NewGuid().ToString("D");
var control = new SaturationControl(directory, run);
var observerPath = Environment.GetEnvironmentVariable("OTOPCUA_SDK2_SATURATION_OBSERVER")!;
var frameworkDependent = observerPath.EndsWith(".dll", StringComparison.OrdinalIgnoreCase);
using var observer = new Process { StartInfo = new ProcessStartInfo(frameworkDependent ? "dotnet" : observerPath)
{ RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false } };
if (frameworkDependent) observer.StartInfo.ArgumentList.Add(observerPath);
observer.StartInfo.ArgumentList.Add(directory);
observer.StartInfo.ArgumentList.Add(run);
using var deadline = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken);
deadline.CancelAfter(TimeSpan.FromSeconds(40));
try
{
observer.Start().ShouldBeTrue();
var stdout = observer.StandardOutput.ReadToEndAsync(deadline.Token);
var stderr = observer.StandardError.ReadToEndAsync(deadline.Token);
await control.AttachAsync(deadline.Token);
if (foreignMarker) SaturationMarkers.Log.Boundary(Guid.NewGuid().ToString("D"), "window1-start", 321, 80);
control.Boundary("window1-start", 321, 80);
await Task.Delay(20, deadline.Token);
control.Boundary("window1-end", 352, 88);
control.Boundary("window2-start", 353, 88);
await Task.Delay(20, deadline.Token);
control.Boundary("window2-end", 384, 96);
await control.DetachAndDrainAsync(deadline.Token);
control.Readiness.ShouldBe(!foreignMarker);
SaturationMarkers.Log.IsEnabled().ShouldBeFalse("Stop and stream drain precede return to the target");
await observer.WaitForExitAsync(deadline.Token);
observer.ExitCode.ShouldBe(0, await stderr);
await stdout;
}
finally
{
if (!observer.HasExited) { observer.Kill(entireProcessTree: true); await observer.WaitForExitAsync(TestContext.Current.CancellationToken); }
Directory.Delete(directory, recursive: true);
}
}
[EventSource(Name = "OtOpcUa-Sdk2-Saturation")]
private sealed class SaturationMarkers : EventSource
{
internal static readonly SaturationMarkers Log = new();
[Event(1, Level = EventLevel.Informational)]
public void Boundary(string runId, string name, int cycle, int restarts) => WriteEvent(1, runId, name, cycle, restarts);
}
private sealed class AcceptWrites : IOpcUaNodeWriteGateway
{
public Task<NodeWriteOutcome> WriteAsync(string id, object? value, AddressSpaceRealm realm, CancellationToken ct) =>