From f4054f6201390791aa238375965f011a5b4358be Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 10 Nov 2026 18:01:41 -0500 Subject: [PATCH] test: enforce late-slope restart and shutdown deadlines --- .../opcua-lifecycle-memory/LATE_SLOPE.md | 9 ++++- .../Sdk2LifecycleLateSlopeTests.cs | 39 ++++++++++++++----- .../Sdk2LifecycleSoakTests.cs | 10 ++++- 3 files changed, 45 insertions(+), 13 deletions(-) diff --git a/scripts/validation/opcua-lifecycle-memory/LATE_SLOPE.md b/scripts/validation/opcua-lifecycle-memory/LATE_SLOPE.md index 2c811fc80..d02ba8d95 100644 --- a/scripts/validation/opcua-lifecycle-memory/LATE_SLOPE.md +++ b/scripts/validation/opcua-lifecycle-memory/LATE_SLOPE.md @@ -36,8 +36,9 @@ observed quiescent RSS and final values. Predeclare these elapsed-time windows: | Final late phase | 6600–7200 | Proposed fixed bounds are startup 120s, each complete cycle including its restart -30s, cycle deadline 7230s, independent shutdown owners 20s each and container -deadline 7440s. The final cycle may finish after 7200s, but cannot extend +30s, cycle deadline 7230s, shutdown ceiling 60s and container deadline 7440s. +The active restart helper has a separate 25s linked deadline; its elapsed time +is recorded before post-restart configuration. The final cycle may finish after 7200s, but cannot extend its 30s bound. Validate these bounds with reviewed short controls before committing the protocol; hitting one makes the diagnostic incomplete, not grounds to relax it. @@ -75,6 +76,10 @@ the generic profile mode currently throws earlier and must keep its old behavior Cleanup must attempt both independent owners under their separate deadlines even when one fails or the operation token is canceled. Preserve the first rejection alongside any later cleanup failure; neither error may erase the other. +The shutdown ceiling covers 5s to capture manager references, 20s for each +owner, 5s to delete the private client directory and 10s to verify shutdown. +Any local deadline or overall elapsed miss is fatal and makes the diagnostic +incomplete; the container deadline is unchanged. Before execution, implement and review the proposed setup/last-cycle/cleanup and outer deadlines in the actual harness and launcher; no extension or retry-to-green. diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2LifecycleLateSlopeTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2LifecycleLateSlopeTests.cs index 309d83c9c..095fc362e 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2LifecycleLateSlopeTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2LifecycleLateSlopeTests.cs @@ -18,6 +18,7 @@ public sealed partial class Sdk2LifecycleSoakTests internal const int StartupDeadline = 120; internal const int PhaseDeadline = 7230; internal const int CycleDeadline = 30; + internal const int ShutdownDeadline = 60; private readonly List _failures = []; private double _shutdownSeconds; @@ -70,7 +71,7 @@ public sealed partial class Sdk2LifecycleSoakTests async Task AttemptAsync(Func? dispose, string owner) { if (dispose is null) return; - try { await dispose().WaitAsync(TimeSpan.FromSeconds(20), CancellationToken.None); } + try { await Task.Run(dispose, CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(20), CancellationToken.None); } catch (Exception error) { Record(new InvalidOperationException($"Late-slope {owner} cleanup failed.", error)); } } } @@ -86,33 +87,51 @@ public sealed partial class Sdk2LifecycleSoakTests { try { - serverManager = rig?.Server.CertificateManager ?? throw new InvalidOperationException("Missing final server manager."); - cores = CertificateCores(ClientManager!).Concat(CertificateCores(serverManager)).ToArray(); + var captured = await Task.Run(() => + { + var manager = rig?.Server.CertificateManager ?? throw new InvalidOperationException("Missing final server manager."); + return (Manager: manager, Cores: CertificateCores(ClientManager!).Concat(CertificateCores(manager)).ToArray()); + }, CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(5), CancellationToken.None); + serverManager = captured.Manager; + cores = captured.Cores; } catch (Exception error) { Record(error); } } await ReleaseOwnersAsync(rig is null ? null : () => rig.DisposeAsync().AsTask(), application is null ? null : () => application.DisposeAsync().AsTask()); - try { if (Directory.Exists(clientRoot)) Directory.Delete(clientRoot, true); } + try + { + await Task.Run(() => { if (Directory.Exists(clientRoot)) Directory.Delete(clientRoot, true); }, CancellationToken.None) + .WaitAsync(TimeSpan.FromSeconds(5), CancellationToken.None); + } catch (Exception error) { Record(error); } if (PhaseStarted && cores is not null && _failures.Count == failuresBeforeShutdown) { try { - RequiredField(ClientManager!, "m_disposed").ShouldBeTrue(); - RequiredField(serverManager!, "m_disposed").ShouldBeTrue(); - cores.Sum(CachedStores).ShouldBe(0); using var probe = new CancellationTokenSource(TimeSpan.FromSeconds(10)); - (await DescriptorCountsAsync(probe.Token, clientRoot, serverRoot!)).Pki.ShouldBe(0); - Directory.Exists(clientRoot).ShouldBeFalse(); - Directory.Exists(serverRoot!).ShouldBeFalse(); + var probeToken = probe.Token; + await Task.Run(async () => + { + RequiredField(ClientManager!, "m_disposed").ShouldBeTrue(); + RequiredField(serverManager!, "m_disposed").ShouldBeTrue(); + cores.Sum(CachedStores).ShouldBe(0); + (await DescriptorCountsAsync(probeToken, clientRoot, serverRoot!)).Pki.ShouldBe(0); + Directory.Exists(clientRoot).ShouldBeFalse(); + Directory.Exists(serverRoot!).ShouldBeFalse(); + }, CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(10), CancellationToken.None); _shutdownVerified = true; } catch (Exception error) { Record(error); } } _shutdownSeconds = watch.Elapsed.TotalSeconds; + if (_shutdownSeconds > ShutdownDeadline) + { + _shutdownVerified = false; + Record(new TimeoutException("Late-slope shutdown exceeded its fixed 60-second ceiling.")); + } } internal async Task EmitAndPreserveFailuresAsync(StreamWriter evidence, int cycles, int restarts, diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2LifecycleSoakTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2LifecycleSoakTests.cs index f764146ad..f83fd735b 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2LifecycleSoakTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2LifecycleSoakTests.cs @@ -184,21 +184,29 @@ public sealed partial class Sdk2LifecycleSoakTests if (cycles % 4 == 0) { var restart = Stopwatch.StartNew(); + double? boundedRestartSeconds = null; if (lateSlope is null) await RestartWithActiveResourcesAsync(rig, config, cycleToken).WaitAsync(TimeSpan.FromSeconds(25), cycleToken); else { using var restartDeadline = LateSlopeRun.StartLinkedDeadline(cycleToken, TimeSpan.FromSeconds(25)); + var boundedRestart = Stopwatch.StartNew(); try { await RestartWithActiveResourcesAsync(rig, config, restartDeadline.Token); restartDeadline.Token.ThrowIfCancellationRequested(); + boundedRestartSeconds = boundedRestart.Elapsed.TotalSeconds; + boundedRestartSeconds.Value.ShouldBeLessThanOrEqualTo(25); } finally { restartDeadline.Cancel(); } } Configure(rig); restarts++; - await LogAsync(evidence, new { kind = "restart", cycles, restarts, seconds = restart.Elapsed.TotalSeconds }); + if (lateSlope is null) + await LogAsync(evidence, new { kind = "restart", cycles, restarts, seconds = restart.Elapsed.TotalSeconds }); + else + await LogAsync(evidence, new { kind = "restart", cycles, restarts, seconds = restart.Elapsed.TotalSeconds, + boundedRestartSeconds = boundedRestartSeconds!.Value }); } if (saturation is not null) {