using System.Diagnostics; using System.Security.Cryptography; using System.Text; using Serilog; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Core.Scripting; using ZB.MOM.WW.OtOpcUa.Core.Scripting.OutOfProcess; using ZB.MOM.WW.OtOpcUa.Core.Scripting.Protocol.V1; namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests; /// /// lmxopcua#789 review: the blocking pipe I/O runs the host's reader on a dedicated thread and blocks producers on a /// full pipe. A fault in the reader's inline completion work must end that worker, not the server process; and a worker /// that stops reading while it keeps heartbeating must be killed so blocked producers are released. /// [Collection(ScriptWorkerCollection.Name)] [Trait("Category", "ScriptWorker")] public sealed class PipeFaultTests { private const string Healthy = "return 7;"; private const string Runaway = "while (true) { }"; private static CancellationToken Ct => TestContext.Current.CancellationToken; /// /// The slot release runs on the pipe reader, inline, before the caller resumes. When it throws, the reader must /// end the worker as a fault: the run in flight beside it answers BadResourceUnavailable ("worker restarted"), /// the shard restarts, and the next evaluation is served — instead of an unhandled exception on a dedicated thread, /// which would terminate the host. /// [Fact] public async Task A_completion_callback_that_throws_on_the_reader_ends_that_worker_as_faulted_not_the_host() { await using var runner = Scripts.Runner(Scripts.Options()); (await runner.RunAsync(Scripts.Vt("boom", Healthy), Ct)).Value.ShouldBe(7, "positive control: the hook is not armed yet"); var worker = runner.Shards[0].Current.ShouldNotBeNull(); var spin = runner.RunAsync(Scripts.Vt("spin", Runaway, TimeSpan.FromSeconds(30)), Ct); await Scripts.Eventually(() => worker.InFlightCount > 0 ? worker : null, TimeSpan.FromSeconds(30), "the sibling in flight"); var armed = 1; runner.SlotReleasingForTest = key => { if (key.EndsWith("boom", StringComparison.Ordinal) && Interlocked.Exchange(ref armed, 0) == 1) throw new InvalidOperationException("a completion callback failed"); }; await runner.RunAsync(Scripts.Vt("boom", Healthy), Ct); var exit = await worker.Exited.WaitAsync(TimeSpan.FromSeconds(15), Ct); exit.Reason.ShouldBe(WorkerExitReason.ProtocolFault); var sibling = await spin.WaitAsync(TimeSpan.FromSeconds(15), Ct); sibling.Status.ShouldBe(ScriptRunStatus.Unavailable); sibling.Reason.ShouldBe(ScriptRunOutcome.WorkerRestartedReason); sibling.ToVirtualTagEvalResult().StatusCode.ShouldBe(ZB.MOM.WW.OtOpcUa.Commons.OpcUa.OpcUaStatus.BadResourceUnavailable); (await runner.RunAsync(Scripts.Vt("after", Healthy), Ct)).Value.ShouldBe(7, "the shard restarted and serves again"); runner.Shards[0].Current.ShouldNotBeNull().Pid.ShouldNotBe(worker.Pid); } /// /// A worker whose reader stops while its heartbeats go on: the host's writes fill the pipe and the producer (here /// a deploy loading scripts, the case nothing else would end: no evaluation is outstanding and a compile's timeout /// starts only when the worker starts it) blocks in the write. The write-stall check must kill the worker within /// its bound — as unresponsive, not for silence, which proves it kept heartbeating — and that releases the producer. /// [Fact] public async Task A_worker_that_stops_reading_but_keeps_heartbeating_is_killed_and_the_blocked_producer_returns() { var heartbeat = TimeSpan.FromMilliseconds(200); var spec = new WorkerLaunchSpec("t", OutOfProcessScriptRunner.ResolveWorkerPath(new ScriptWorkerOptions { ExecutablePath = Scripts.WorkerUnderTest }), 256L * 1024 * 1024, EvalThreads: 2, HardGrace: TimeSpan.FromSeconds(60), HeartbeatInterval: heartbeat, HandshakeTimeout: TimeSpan.FromSeconds(15), HeartbeatSilenceLimit: TimeSpan.FromSeconds(30), TestStallReaderOnDiag: true); var connection = await WorkerConnection.StartAsync(spec, new LoggerConfiguration().CreateLogger(), _ => { }, _ => { }, Ct); await using var owned = connection; var warm = await connection.EnsureLoadedAsync("k", ContextKind.VirtualTag, Healthy, SHA256.HashData(Encoding.UTF8.GetBytes(Healthy)), []).WaitAsync(TimeSpan.FromSeconds(60), Ct); warm.Outcome.ShouldNotBeNull().Ok.ShouldBeTrue(); connection.SilenceLimit = heartbeat * 5; // armed once the cold start is over (see SupervisionTests) var stallLimit = TimeSpan.FromSeconds(2); connection.WriteStallLimitOverride = stallLimit; _ = connection.DiagnosticsAsync(TimeSpan.FromSeconds(60)); // the worker's reader stops here for good var producer = Task.Factory.StartNew(() => { // About 5 MB of loads: far more than the pipe's buffer and the worker reader's own. for (var i = 0; i < 20; i++) { var source = $"return {i}; // " + new string('x', 256 * 1024); _ = connection.EnsureLoadedAsync($"big{i}", ContextKind.VirtualTag, source, SHA256.HashData(Encoding.UTF8.GetBytes(source)), []); } }, Ct, TaskCreationOptions.LongRunning, TaskScheduler.Default); var sw = Stopwatch.StartNew(); await Task.Delay(stallLimit / 2, Ct); producer.IsCompleted.ShouldBeFalse("positive control: the producer is blocked in a write to the full pipe"); connection.IsAlive.ShouldBeTrue("positive control: heartbeats keep the silence check quiet"); var exit = await connection.Exited.WaitAsync(TimeSpan.FromSeconds(20), Ct); exit.Reason.ShouldBe(WorkerExitReason.KilledUnresponsive, "killed by the write-stall check, not for silence"); await producer.WaitAsync(TimeSpan.FromSeconds(10), Ct); sw.Elapsed.ShouldBeLessThan(stallLimit + TimeSpan.FromSeconds(8), "the producer was released within the bound"); } }