lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/SeamEndToEndTests.cs

using System.Collections.Concurrent;
using System.Diagnostics;
using Serilog;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
using ZB.MOM.WW.OtOpcUa.Driver.Calculation;

namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests;

/// <summary>
///     lmxopcua#780 — the whole path a client sees: a real Calculation driver whose seam runs out of process on the real
///     worker. A script failure publishes <c>BadInternalError</c> as today; the worker dying under a script publishes
///     <c>BadResourceUnavailable</c> (0x80040000), the ScadaBridge-visible code; and the tag recovers Good afterwards.
/// </summary>
[Collection(ScriptWorkerCollection.Name)]
[Trait("Category", "ScriptWorker")]
public sealed class SeamEndToEndTests
{
    private const uint StatusGood = 0u;
    private const uint StatusBadInternalError = 0x80020000u;
    private const uint StatusBadResourceUnavailable = 0x80040000u;
    private const string Tag = "calc/e/spin";

    /// <summary>Spins for <c>src/ms</c> milliseconds, throws when it is negative, and returns it otherwise.</summary>
    private const string Script =
        "var ms = (double)ctx.GetTag(\"src/ms\").Value; if (ms < 0) throw new System.InvalidOperationException(\"negative\"); " +
        "var end = ctx.Now.AddMilliseconds(ms); while (ctx.Now < end) { } return ms;";

    private static CancellationToken Ct => TestContext.Current.CancellationToken;

    private static async Task Settle(CalculationDriver driver)
    {
        var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(60);
        while (DateTime.UtcNow < deadline && !driver.IsEvaluationIdle) await Task.Delay(10);
        driver.IsEvaluationIdle.ShouldBeTrue();
    }

    [Fact]
    public async Task A_calculation_tag_publishes_BadResourceUnavailable_only_when_the_worker_died_under_it()
    {
        await using var runner = Scripts.Runner(Scripts.Options());
        var json = $"{{\"scriptId\":\"s\",\"changeTriggered\":true,\"scriptSource\":{System.Text.Json.JsonSerializer.Serialize(Script)}}}";
        var driver = new CalculationDriver(
            new CalculationDriverOptions { RawTags = [new RawTagEntry(Tag, json, WriteIdempotent: false)], RunTimeout = TimeSpan.FromSeconds(20), ScriptRunner = runner },
            "calc-drv", new ScriptRootLogger(new LoggerConfiguration().CreateLogger()));
        var published = new ConcurrentQueue<DataChangeEventArgs>();
        driver.OnDataChange += (_, e) => published.Enqueue(e);
        await driver.InitializeAsync("{}", Ct);

        driver.OnDependencyValue("src/ms", 0.0, StatusGood, DateTime.UtcNow);
        await Settle(driver);
        published.TryDequeue(out var good).ShouldBeTrue();
        good!.Snapshot.StatusCode.ShouldBe(StatusGood);
        good.Snapshot.Value.ShouldBe(0.0);

        // A script fault: BadInternalError, exactly as in process.
        driver.OnDependencyValue("src/ms", -1.0, StatusGood, DateTime.UtcNow);
        await Settle(driver);
        published.TryDequeue(out var threw).ShouldBeTrue();
        threw!.Snapshot.StatusCode.ShouldBe(StatusBadInternalError);

        driver.OnDependencyValue("src/ms", 1.0, StatusGood, DateTime.UtcNow);
        await Settle(driver);
        published.TryDequeue(out var back).ShouldBeTrue();
        back!.Snapshot.StatusCode.ShouldBe(StatusGood);

        // The worker dies under a run: the runner, not the script, failed.
        driver.OnDependencyValue("src/ms", 5000.0, StatusGood, DateTime.UtcNow);
        var worker = await Scripts.Eventually(() => runner.Shards[0].Current is { InFlightCount: 1 } w ? w : null,
            TimeSpan.FromSeconds(10), "the run in flight");
        await Task.Delay(200, Ct);
        Process.GetProcessById(worker.Pid).Kill();
        await Settle(driver);
        published.TryDequeue(out var unavailable).ShouldBeTrue();
        unavailable!.Snapshot.StatusCode.ShouldBe(StatusBadResourceUnavailable);
        unavailable.Snapshot.Value.ShouldBe(1.0, "the last good value rides out with the Bad status");

        // Recovery: the next trigger runs (in the isolation worker, as a suspect) and publishes Good.
        driver.OnDependencyValue("src/ms", 2.0, StatusGood, DateTime.UtcNow);
        await Settle(driver);
        published.TryDequeue(out var recovered).ShouldBeTrue();
        recovered!.Snapshot.StatusCode.ShouldBe(StatusGood);
        recovered.Snapshot.Value.ShouldBe(2.0);
        await driver.ShutdownAsync(Ct);
    }
}