using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.DependencyInjection; using NSubstitute; using ZB.MOM.WW.ScadaBridge.CentralUI.ScriptAnalysis; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services; using ZB.MOM.WW.ScadaBridge.Commons.Types; namespace ZB.MOM.WW.ScadaBridge.CentralUI.Tests.ScriptAnalysis; [Collection("Sandbox worker processes")] public class SandboxWorkerIsolationTests { [Fact] public async Task FourBusyWorkers_RejectQueuedFifthWithoutStartingAnotherProcess() { var allEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var count = 0; var external = Substitute.For(); external.CallAsync("probe", "blocked", Arg.Any?>(), Arg.Any()) .Returns(_ => { if (Interlocked.Increment(ref count) == 4) allEntered.TrySetResult(); return release.Task; }); using var services = new ServiceCollection().AddSingleton(external).BuildServiceProvider(); var catalog = Substitute.For(); var workers = Enumerable.Range(0, 4).Select(_ => SandboxWorkerProtocol.RunAsync(new SandboxRunRequest( "await ExternalSystem.Call(\"probe\", \"blocked\"); return 42;", null, 10), services, catalog, CancellationToken.None)).ToArray(); try { await allEntered.Task.WaitAsync(TimeSpan.FromSeconds(8)); var fifthStarted = false; var fifth = await SandboxWorkerProtocol.RunCoreAsync(new SandboxRunRequest("return 1;", null, 1), services, catalog, CancellationToken.None, _ => fifthStarted = true); Assert.Equal(SandboxErrorKind.Timeout, fifth.ErrorKind); Assert.False(fifthStarted); } finally { release.TrySetResult(new ExternalCallResult(true, "null", null)); } foreach (var result in await Task.WhenAll(workers)) Assert.True(result.Success, result.Error); } [Fact] public async Task UncooperativeBroker_StillKillsChildAtDeadline_AndKeepsLifetimeUntilBrokerCompletes() { var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var external = Substitute.For(); external.CallAsync("probe", "blocked", Arg.Any?>(), Arg.Any()) .Returns(_ => { entered.TrySetResult(); return release.Task; }); using var services = new ServiceCollection().AddSingleton(external).BuildServiceProvider(); System.Diagnostics.Process? child = null; var run = SandboxWorkerProtocol.RunCoreAsync(new SandboxRunRequest( "await ExternalSystem.Call(\"probe\", \"blocked\"); return 42;", null, 10), services, Substitute.For(), CancellationToken.None, id => child = System.Diagnostics.Process.GetProcessById(id)); try { await entered.Task.WaitAsync(TimeSpan.FromSeconds(10)); Assert.NotNull(child); await child.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(12)); Assert.False(run.IsCompleted); // Do not dispose the broker's request scope or recycle its charged slot. } finally { release.TrySetResult(new ExternalCallResult(true, "null", null)); } var result = await run.WaitAsync(TimeSpan.FromSeconds(5)); Assert.Equal(SandboxErrorKind.Timeout, result.ErrorKind); child?.Dispose(); } [Fact] public async Task NoncooperativeWorker_IsTerminated_AndCapacityCanBeReused() { var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var external = Substitute.For(); external.CallAsync("probe", "entered", Arg.Any?>(), Arg.Any()) .Returns(_ => { entered.TrySetResult(); return new ExternalCallResult(true, "null", null); }); var services = new ServiceCollection().AddSingleton(external).BuildServiceProvider(); using var cache = new MemoryCache(new MemoryCacheOptions()); var runner = new ScriptAnalysisService(Substitute.For(), cache, services); var run = runner.RunInSandboxAsync(new SandboxRunRequest( "await ExternalSystem.Call(\"probe\", \"entered\"); while (true) { }", null, 10), CancellationToken.None); await entered.Task.WaitAsync(TimeSpan.FromSeconds(10)); var result = await run.WaitAsync(TimeSpan.FromSeconds(15)); Assert.Equal(SandboxErrorKind.Timeout, result.ErrorKind); var next = await runner.RunInSandboxAsync(new SandboxRunRequest("return 42;", null, 10), CancellationToken.None); Assert.True(next.Success, next.Error); Assert.Equal("42", next.ReturnValueJson); Assert.Single(external.ReceivedCalls()); } }