ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.CentralUI.Tests/ScriptAnalysis/SandboxWorkerIsolationTests.cs

using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.DependencyInjection;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.CentralUI.ScriptAnalysis;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Types;

namespace ZB.MOM.WW.ScadaBridge.CentralUI.Tests.ScriptAnalysis;

[Collection("Sandbox worker processes")]
public class SandboxWorkerIsolationTests
{
    /// <summary>Verifies that four busy workers reject a queued fifth request without starting another process.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task FourBusyWorkers_RejectQueuedFifthWithoutStartingAnotherProcess()
    {
        var allEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
        var release = new TaskCompletionSource<ExternalCallResult>(TaskCreationOptions.RunContinuationsAsynchronously);
        var count = 0;
        var external = Substitute.For<IExternalSystemClient>();
        external.CallAsync("probe", "blocked", Arg.Any<IReadOnlyDictionary<string, object?>?>(), Arg.Any<CancellationToken>())
            .Returns(_ => { if (Interlocked.Increment(ref count) == 4) allEntered.TrySetResult(); return release.Task; });
        using var services = new ServiceCollection().AddSingleton(external).BuildServiceProvider();
        var catalog = Substitute.For<ISharedScriptCatalog>();
        var workers = Enumerable.Range(0, 4).Select(_ => SandboxWorkerProtocol.RunAsync(new SandboxRunRequest(
            "await ExternalSystem.Call(\"probe\", \"blocked\"); return 42;", null, 10), services, catalog, CancellationToken.None)).ToArray();
        try
        {
            await allEntered.Task.WaitAsync(TimeSpan.FromSeconds(8));
            var fifthStarted = false;
            var fifth = await SandboxWorkerProtocol.RunCoreAsync(new SandboxRunRequest("return 1;", null, 1),
                services, catalog, CancellationToken.None, _ => fifthStarted = true);
            Assert.Equal(SandboxErrorKind.Timeout, fifth.ErrorKind);
            Assert.False(fifthStarted);
        }
        finally { release.TrySetResult(new ExternalCallResult(true, "null", null)); }
        foreach (var result in await Task.WhenAll(workers)) Assert.True(result.Success, result.Error);
    }

    /// <summary>Verifies that an uncooperative broker still has its child killed at the deadline and keeps the lifetime until the broker completes.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task UncooperativeBroker_StillKillsChildAtDeadline_AndKeepsLifetimeUntilBrokerCompletes()
    {
        var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
        var release = new TaskCompletionSource<ExternalCallResult>(TaskCreationOptions.RunContinuationsAsynchronously);
        var external = Substitute.For<IExternalSystemClient>();
        external.CallAsync("probe", "blocked", Arg.Any<IReadOnlyDictionary<string, object?>?>(), Arg.Any<CancellationToken>())
            .Returns(_ => { entered.TrySetResult(); return release.Task; });
        using var services = new ServiceCollection().AddSingleton(external).BuildServiceProvider();
        System.Diagnostics.Process? child = null;
        var run = SandboxWorkerProtocol.RunCoreAsync(new SandboxRunRequest(
            "await ExternalSystem.Call(\"probe\", \"blocked\"); return 42;", null, 10), services,
            Substitute.For<ISharedScriptCatalog>(), CancellationToken.None,
            id => child = System.Diagnostics.Process.GetProcessById(id));
        try
        {
            await entered.Task.WaitAsync(TimeSpan.FromSeconds(10));
            Assert.NotNull(child);
            await child.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(12));
            Assert.False(run.IsCompleted); // Do not dispose the broker's request scope or recycle its charged slot.
        }
        finally { release.TrySetResult(new ExternalCallResult(true, "null", null)); }
        var result = await run.WaitAsync(TimeSpan.FromSeconds(5));
        Assert.Equal(SandboxErrorKind.Timeout, result.ErrorKind);
        child?.Dispose();
    }

    /// <summary>Verifies that a non-cooperative worker is terminated and its capacity can be reused.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task NoncooperativeWorker_IsTerminated_AndCapacityCanBeReused()
    {
        var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
        var external = Substitute.For<IExternalSystemClient>();
        external.CallAsync("probe", "entered", Arg.Any<IReadOnlyDictionary<string, object?>?>(), Arg.Any<CancellationToken>())
            .Returns(_ => { entered.TrySetResult(); return new ExternalCallResult(true, "null", null); });
        var services = new ServiceCollection().AddSingleton(external).BuildServiceProvider();
        using var cache = new MemoryCache(new MemoryCacheOptions());
        var runner = new ScriptAnalysisService(Substitute.For<ISharedScriptCatalog>(), cache, services);
        var run = runner.RunInSandboxAsync(new SandboxRunRequest(
            "await ExternalSystem.Call(\"probe\", \"entered\"); while (true) { }", null, 10), CancellationToken.None);
        await entered.Task.WaitAsync(TimeSpan.FromSeconds(10));
        var result = await run.WaitAsync(TimeSpan.FromSeconds(15));
        Assert.Equal(SandboxErrorKind.Timeout, result.ErrorKind);
        var next = await runner.RunInSandboxAsync(new SandboxRunRequest("return 42;", null, 10), CancellationToken.None);
        Assert.True(next.Success, next.Error);
        Assert.Equal("42", next.ReturnValueJson);
        Assert.Single(external.ReceivedCalls());
    }
}