lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ShadowModeTests.cs

using System.Collections.Concurrent;
using System.Diagnostics;
using Microsoft.Extensions.Logging.Abstractions;
using Serilog;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
using ZB.MOM.WW.OtOpcUa.Driver.Calculation;

namespace ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests;

/// <summary>
///     lmxopcua#780 P2 — <c>Scripting:Runner=Shadow</c> over the REAL worker: the seams publish their in-process results
///     and mirror each compiled evaluation to the worker pool. Ordinary scripts compare equal; a deliberately divergent
///     one (it reads which process it runs in) and an unsupported return type (design §14 #11) are counted as mismatches
///     for exactly those scripts; and a runaway that only runs away in the worker is killed and quarantined THERE while
///     every published value stays the in-process one. Alarms: the transition sequence is the in-process engine's, a
///     killed shadow worker never annotates or clears a condition.
/// </summary>
[Collection(ScriptWorkerCollection.Name)]
[Trait("Category", "ScriptWorker")]
public sealed class ShadowModeTests
{
    /// <summary>True only inside the ScriptWorker (its base directory is <c>script-worker/</c>): a test-only divergence.</summary>
    private const string InWorker = "System.AppContext.BaseDirectory.Contains(\"script-worker\")";

    private static CancellationToken Ct => TestContext.Current.CancellationToken;

    private static ScriptingOptions ShadowOptions(TimeSpan? hardGrace = null)
    {
        var options = Scripts.Options(hardGrace: hardGrace ?? TimeSpan.FromMilliseconds(500), quarantineAfterKills: 2);
        options.Runner = ScriptRunnerMode.Shadow;
        return options;
    }

    private static CalculationEvaluator Calc(IScriptRunner? runner, TimeSpan? budget = null)
        => new(NullLogger.Instance, new ScriptRootLogger(new LoggerConfiguration().CreateLogger()),
            budget ?? TimeSpan.FromSeconds(5), scriptRunner: runner);

    private static Dictionary<string, DataValueSnapshot> Inputs(int i)
    {
        var now = DateTime.UtcNow;
        DataValueSnapshot S(object v) => new(v, 0u, now, now);
        return new Dictionary<string, DataValueSnapshot>(StringComparer.Ordinal)
        {
            ["s"] = S((short)(5 + i)),
            ["d"] = S(1.25 * i),
            ["str"] = S("abc" + i),
            ["i"] = S(i),
            ["t"] = S(new DateTime(2026, 11, 10, 12, 0, 0, DateTimeKind.Utc).AddSeconds(i)),
            ["arr"] = S(new[] { 1.5, 2.5 * i }),
        };
    }

    private static bool Same(VirtualTagEvalResult a, VirtualTagEvalResult b)
        => a.Success == b.Success && a.Reason == b.Reason && a.StatusCode == b.StatusCode
           && System.Collections.StructuralComparisons.StructuralEqualityComparer.Equals(a.Value, b.Value)
           && a.Value?.GetType() == b.Value?.GetType();

    [Fact]
    public async Task Ordinary_scripts_match_and_a_divergent_and_an_unsupported_one_are_counted_against_exactly_those_scripts()
    {
        var log = new CapturingSink();
        await using var pool = Scripts.Runner(ShadowOptions());
        using var shadow = new ShadowScriptRunner(pool, ShadowOptions(), log.CreateLogger());
        using var shadowed = Calc(shadow);
        using var reference = Calc(null);
        shadowed.SubmitsBatches.ShouldBeFalse("Shadow keeps the in-process one-at-a-time pump, not the runner path");

        var scripts = new Dictionary<string, string>(StringComparer.Ordinal)
        {
            ["short"] = "return (short)ctx.GetTag(\"s\").Value + 1;",
            ["double"] = "return (double)ctx.GetTag(\"d\").Value * 2.5;",
            ["string"] = "return \"v=\" + ctx.GetTag(\"str\").Value;",
            ["intArray"] = "return new[] { (int)ctx.GetTag(\"i\").Value, 2 };",
            ["doubleArray"] = "var a = (double[])ctx.GetTag(\"arr\").Value; return new[] { a[1], a[0] };",
            ["datetime"] = "return ((DateTime)ctx.GetTag(\"t\").Value).AddSeconds(1);",
            ["bool"] = "return (int)ctx.GetTag(\"i\").Value > 1;",
            ["quality"] = "return ctx.GetTag(\"i\").StatusCode;",
            ["throws"] = "if ((int)ctx.GetTag(\"i\").Value >= 0) throw new InvalidOperationException(\"boom \" + ctx.GetTag(\"i\").Value); return 0;",
            ["divergent"] = $"return {InWorker} ? 2 : 1;",
            ["decimal"] = "return 1.5m;",
        };

        for (var i = 0; i < 3; i++)
        {
            foreach (var (key, source) in scripts)
            {
                var published = await shadowed.EvaluateAsync(key, source, Inputs(i), Ct);
                var expected = await reference.EvaluateAsync(key, source, Inputs(i), Ct);
                Same(published, expected).ShouldBeTrue($"{key}: Shadow publishes exactly what InProcess publishes ({published} vs {expected})");
            }
            (await shadow.WaitIdleAsync(TimeSpan.FromSeconds(60))).ShouldBeTrue();
        }

        var stats = shadow.Stats();
        stats.Dropped.ShouldBe(0);
        stats.ShadowUnavailable.ShouldBe(0);
        stats.Compared.ShouldBe(scripts.Count * 3);
        stats.Mismatched.ShouldBe(6, string.Join(Environment.NewLine, shadow.Mismatches.Select(m => $"{m.Key}: {m.LastReason} — {m.LastSummary}")));
        var mismatches = shadow.Mismatches.ToDictionary(m => m.Key);
        mismatches.Keys.ShouldBe(["decimal", "divergent"], ignoreOrder: true);
        mismatches["divergent"].LastReason.ShouldBe("value");
        mismatches["divergent"].LastSummary.ShouldBe("published Ok Int32 1; shadow Ok Int32 2");
        mismatches["divergent"].Count.ShouldBe(3);
        mismatches["decimal"].LastReason.ShouldBe("unsupported_type");
        mismatches["decimal"].LastSummary.ShouldContain("System.Decimal");

        var lines = log.Events.Where(e => e.MessageTemplate.Text.StartsWith("Script shadow mismatch", StringComparison.Ordinal)).ToArray();
        lines.Length.ShouldBe(2, "one line per script and reason");
        lines.ShouldAllBe(e => e.Level == Serilog.Events.LogEventLevel.Warning);
    }

    [Fact]
    public async Task A_runaway_that_runs_away_only_in_the_worker_is_killed_and_quarantined_there_and_never_reaches_a_published_value()
    {
        var log = new CapturingSink();
        await using var pool = Scripts.Runner(ShadowOptions(TimeSpan.FromMilliseconds(400)), log.CreateLogger());
        using var shadow = new ShadowScriptRunner(pool, ShadowOptions(), log.CreateLogger());
        using var shadowed = Calc(shadow, TimeSpan.FromMilliseconds(300));
        const string runaway = $"if ({InWorker}) {{ while (true) {{ }} }} return (int)ctx.GetTag(\"i\").Value * 2;";
        const string sibling = "return (int)ctx.GetTag(\"i\").Value + 100;";

        var slowest = TimeSpan.Zero;
        var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(90);
        var i = 0;
        for (; !pool.QuarantinedKeys(ScriptContextKind.Calculation).Contains("runaway"); i++)
        {
            if (DateTime.UtcNow > deadline)
                throw new TimeoutException("the shadow never quarantined the runaway:" + Environment.NewLine + string.Join(Environment.NewLine, log.Rendered));
            var sw = Stopwatch.StartNew();
            var a = await shadowed.EvaluateAsync("runaway", runaway, Inputs(i), Ct);
            var b = await shadowed.EvaluateAsync("sibling", sibling, Inputs(i), Ct);
            if (sw.Elapsed > slowest) slowest = sw.Elapsed;
            a.ShouldBe(VirtualTagEvalResult.Ok(i * 2), "published from the in-process run");
            b.ShouldBe(VirtualTagEvalResult.Ok(i + 100));
            await Task.Delay(100, Ct);
        }

        // Still publishing correctly after the shadow quarantined it, and never quarantined on the published side.
        for (var j = i; j < i + 5; j++)
            (await shadowed.EvaluateAsync("runaway", runaway, Inputs(j), Ct)).ShouldBe(VirtualTagEvalResult.Ok(j * 2));
        shadowed.QuarantinedScripts.ShouldBeEmpty();
        slowest.ShouldBeLessThan(TimeSpan.FromMilliseconds(250), "a published evaluation never waits for its shadow");
        log.Rendered.ShouldContain(l => l.Contains("exceeded its hard deadline"));

        (await shadow.WaitIdleAsync(TimeSpan.FromSeconds(30))).ShouldBeTrue();
        var record = shadow.Mismatches.ShouldHaveSingleItem("the sibling on the killed shard is a victim (not compared), never a mismatch");
        record.Key.ShouldBe("runaway");
        record.LastReason.ShouldBe("status");
        record.LastSummary.ShouldContain("shadow Failed [budget]");
        shadow.Stats().DroppedKeyBusy.ShouldBeGreaterThan(0, "triggers while its shadow run was still going were dropped, not queued");
    }

    private static ScriptedAlarmDefinition Alarm(string id, string predicate) =>
        new(AlarmId: id, EquipmentPath: "Plant/Line1/Reactor", AlarmName: id, Kind: AlarmKind.AlarmCondition,
            Severity: AlarmSeverity.High, MessageTemplate: "condition", PredicateScriptSource: predicate);

    private static ScriptedAlarmEngine Engine(TestUpstream up, IScriptRunner? runner)
    {
        var logger = new LoggerConfiguration().CreateLogger();
        return new ScriptedAlarmEngine(up, new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger,
            scriptTimeout: TimeSpan.FromSeconds(2), scriptRunner: runner);
    }

    [Fact]
    public async Task Alarms_in_shadow_transition_exactly_as_in_process_and_a_killed_shadow_worker_never_touches_a_condition()
    {
        await using var pool = Scripts.Runner(ShadowOptions());
        using var shadow = new ShadowScriptRunner(pool, ShadowOptions(), new LoggerConfiguration().CreateLogger());
        var up = new TestUpstream();
        var refUp = new TestUpstream();
        up.Set("x", 0);
        refUp.Set("x", 0);
        using var engine = Engine(up, shadow);
        using var reference = Engine(refUp, null);
        engine.ScriptRunner.ShouldBeNull("Shadow keeps the in-process engine path");
        var events = new ConcurrentQueue<ScriptedAlarmEvent>();
        var refEvents = new ConcurrentQueue<ScriptedAlarmEvent>();
        engine.OnEvent += (_, e) => events.Enqueue(e);
        reference.OnEvent += (_, e) => refEvents.Enqueue(e);
        ScriptedAlarmDefinition[] defs =
        [
            Alarm("high", "return (int)ctx.GetTag(\"x\").Value > 5;"),
            Alarm("divergent", $"return {InWorker};"),
        ];
        await engine.LoadAsync(defs, Ct);
        await reference.LoadAsync(defs, Ct);

        async Task Push(int x)
        {
            up.Push("x", x);
            refUp.Push("x", x);
            await Scripts.Eventually(() => events.Count == refEvents.Count, TimeSpan.FromSeconds(10), "both engines' emissions");
            await Task.Delay(100, Ct);
        }

        await Push(10);
        await Push(2);
        (await shadow.WaitIdleAsync(TimeSpan.FromSeconds(30))).ShouldBeTrue();

        // kill -9 the shadow's worker: the published engine neither annotates nor clears anything because of it.
        var worker = pool.Shards[0].Current.ShouldNotBeNull();
        Process.GetProcessById(worker.Pid).Kill();
        await Push(12);
        await Push(0);
        (await shadow.WaitIdleAsync(TimeSpan.FromSeconds(30))).ShouldBeTrue();
        await Push(9);
        (await shadow.WaitIdleAsync(TimeSpan.FromSeconds(30))).ShouldBeTrue();

        events.Select(e => (e.AlarmId, e.Emission, e.WorstInputStatusCode))
            .ShouldBe(refEvents.Select(e => (e.AlarmId, e.Emission, e.WorstInputStatusCode)), "the in-process engine's sequence, exactly");
        events.Count(e => e is { AlarmId: "high", Emission: EmissionKind.Activated }).ShouldBe(3);
        events.ShouldNotContain(e => e.WorstInputStatusCode != 0u, "no shadow-side failure annotates a published condition");
        engine.GetState("divergent")!.Active.ShouldBe(AlarmActiveState.Inactive, "the in-process predicate (false) drives it");

        var record = shadow.Mismatches.ShouldHaveSingleItem();
        record.Key.ShouldBe("divergent");
        record.Kind.ShouldBe(ScriptContextKind.AlarmPredicate);
        record.LastSummary.ShouldBe("published Ok Boolean False; shadow Ok Boolean True");
        shadow.Stats().Matched.ShouldBeGreaterThan(0);
    }
}