dohertj2/lmxopcua
1
0
Fork 0
Code Issues 26 Pull Requests Actions 2 Packages Projects Releases Wiki Activity

test: enforce late-slope restart and shutdown deadlines

Browse Source
This commit is contained in:
Joseph Doherty
2026-11-10 18:01:41 -05:00
parent 613369fc54
commit f4054f6201
3 changed files with 45 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/validation/opcua-lifecycle-memory/LATE_SLOPE.md
+7 -2
View File
@@ -36,8 +36,9 @@ observed quiescent RSS and final values. Predeclare these elapsed-time windows:
| Final late phase | 6600–7200 |
Proposed fixed bounds are startup 120s, each complete cycle including its restart
30s, cycle deadline 7230s, independent shutdown owners 20s each and container
deadline 7440s. The final cycle may finish after 7200s, but cannot extend
30s, cycle deadline 7230s, shutdown ceiling 60s and container deadline 7440s.
The active restart helper has a separate 25s linked deadline; its elapsed time
is recorded before post-restart configuration. The final cycle may finish after 7200s, but cannot extend
its 30s bound. Validate these bounds with reviewed short controls before committing
the protocol; hitting one makes the diagnostic incomplete, not grounds to relax it.
@@ -75,6 +76,10 @@ the generic profile mode currently throws earlier and must keep its old behavior
Cleanup must attempt both independent owners under their separate deadlines even
when one fails or the operation token is canceled. Preserve the first rejection
alongside any later cleanup failure; neither error may erase the other.
The shutdown ceiling covers 5s to capture manager references, 20s for each
owner, 5s to delete the private client directory and 10s to verify shutdown.
Any local deadline or overall elapsed miss is fatal and makes the diagnostic
incomplete; the container deadline is unchanged.
Before execution, implement and review the proposed setup/last-cycle/cleanup and
outer deadlines in the actual harness and launcher; no extension or retry-to-green.
tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2LifecycleLateSlopeTests.cs
+29 -10
View File
@@ -18,6 +18,7 @@ public sealed partial class Sdk2LifecycleSoakTests
internal const int StartupDeadline = 120;
internal const int PhaseDeadline = 7230;
internal const int CycleDeadline = 30;
internal const int ShutdownDeadline = 60;
private readonly List<ExceptionDispatchInfo> _failures = [];
private double _shutdownSeconds;
@@ -70,7 +71,7 @@ public sealed partial class Sdk2LifecycleSoakTests
async Task AttemptAsync(Func<Task>? dispose, string owner)
{
if (dispose is null) return;
try { await dispose().WaitAsync(TimeSpan.FromSeconds(20), CancellationToken.None); }
try { await Task.Run(dispose, CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(20), CancellationToken.None); }
catch (Exception error) { Record(new InvalidOperationException($"Late-slope {owner} cleanup failed.", error)); }
}
}
@@ -86,33 +87,51 @@ public sealed partial class Sdk2LifecycleSoakTests
{
try
{
serverManager = rig?.Server.CertificateManager ?? throw new InvalidOperationException("Missing final server manager.");
cores = CertificateCores(ClientManager!).Concat(CertificateCores(serverManager)).ToArray();
var captured = await Task.Run(() =>
{
var manager = rig?.Server.CertificateManager ?? throw new InvalidOperationException("Missing final server manager.");
return (Manager: manager, Cores: CertificateCores(ClientManager!).Concat(CertificateCores(manager)).ToArray());
}, CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(5), CancellationToken.None);
serverManager = captured.Manager;
cores = captured.Cores;
}
catch (Exception error) { Record(error); }
}
await ReleaseOwnersAsync(rig is null ? null : () => rig.DisposeAsync().AsTask(),
application is null ? null : () => application.DisposeAsync().AsTask());
try { if (Directory.Exists(clientRoot)) Directory.Delete(clientRoot, true); }
try
{
await Task.Run(() => { if (Directory.Exists(clientRoot)) Directory.Delete(clientRoot, true); }, CancellationToken.None)
.WaitAsync(TimeSpan.FromSeconds(5), CancellationToken.None);
}
catch (Exception error) { Record(error); }
if (PhaseStarted && cores is not null && _failures.Count == failuresBeforeShutdown)
{
try
{
RequiredField<bool>(ClientManager!, "m_disposed").ShouldBeTrue();
RequiredField<bool>(serverManager!, "m_disposed").ShouldBeTrue();
cores.Sum(CachedStores).ShouldBe(0);
using var probe = new CancellationTokenSource(TimeSpan.FromSeconds(10));
(await DescriptorCountsAsync(probe.Token, clientRoot, serverRoot!)).Pki.ShouldBe(0);
Directory.Exists(clientRoot).ShouldBeFalse();
Directory.Exists(serverRoot!).ShouldBeFalse();
var probeToken = probe.Token;
await Task.Run(async () =>
{
RequiredField<bool>(ClientManager!, "m_disposed").ShouldBeTrue();
RequiredField<bool>(serverManager!, "m_disposed").ShouldBeTrue();
cores.Sum(CachedStores).ShouldBe(0);
(await DescriptorCountsAsync(probeToken, clientRoot, serverRoot!)).Pki.ShouldBe(0);
Directory.Exists(clientRoot).ShouldBeFalse();
Directory.Exists(serverRoot!).ShouldBeFalse();
}, CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(10), CancellationToken.None);
_shutdownVerified = true;
}
catch (Exception error) { Record(error); }
}
_shutdownSeconds = watch.Elapsed.TotalSeconds;
if (_shutdownSeconds > ShutdownDeadline)
{
_shutdownVerified = false;
Record(new TimeoutException("Late-slope shutdown exceeded its fixed 60-second ceiling."));
}
}
internal async Task EmitAndPreserveFailuresAsync(StreamWriter evidence, int cycles, int restarts,
tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2LifecycleSoakTests.cs
+9 -1
View File
@@ -184,21 +184,29 @@ public sealed partial class Sdk2LifecycleSoakTests
if (cycles % 4 == 0)
{
var restart = Stopwatch.StartNew();
double? boundedRestartSeconds = null;
if (lateSlope is null)
await RestartWithActiveResourcesAsync(rig, config, cycleToken).WaitAsync(TimeSpan.FromSeconds(25), cycleToken);
else
{
using var restartDeadline = LateSlopeRun.StartLinkedDeadline(cycleToken, TimeSpan.FromSeconds(25));
var boundedRestart = Stopwatch.StartNew();
try
{
await RestartWithActiveResourcesAsync(rig, config, restartDeadline.Token);
restartDeadline.Token.ThrowIfCancellationRequested();
boundedRestartSeconds = boundedRestart.Elapsed.TotalSeconds;
boundedRestartSeconds.Value.ShouldBeLessThanOrEqualTo(25);
}
finally { restartDeadline.Cancel(); }
}
Configure(rig);
restarts++;
await LogAsync(evidence, new { kind = "restart", cycles, restarts, seconds = restart.Elapsed.TotalSeconds });
if (lateSlope is null)
await LogAsync(evidence, new { kind = "restart", cycles, restarts, seconds = restart.Elapsed.TotalSeconds });
else
await LogAsync(evidence, new { kind = "restart", cycles, restarts, seconds = restart.Elapsed.TotalSeconds,
boundedRestartSeconds = boundedRestartSeconds!.Value });
}
if (saturation is not null)
{