using System.Data.Common; using Microsoft.Data.Sqlite; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using NSubstitute; using ZB.MOM.WW.Audit; using ZB.MOM.WW.ScadaBridge.AuditLog.Configuration; using ZB.MOM.WW.ScadaBridge.Commons.Entities.InboundApi; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services; using ZB.MOM.WW.ScadaBridge.Commons.Messages.InboundApi; using ZB.MOM.WW.ScadaBridge.Commons.Types; using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; using ZB.MOM.WW.ScadaBridge.Commons.Types.Debugging; using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; namespace ZB.MOM.WW.ScadaBridge.InboundAPI.Tests; /// /// WP-3: Tests for script execution on central — timeout, handler dispatch, error handling. /// WP-5: Safe error messages. /// public class InboundScriptExecutorTests { private readonly InboundScriptExecutor _executor; private readonly RouteHelper _route; public InboundScriptExecutorTests() { _executor = new InboundScriptExecutor(NullLogger.Instance, Substitute.For()); var locator = Substitute.For(); var router = Substitute.For(); _route = new RouteHelper(locator, router); } [Fact] public async Task ExecuteAsync_TraceFlag_ReachesTheRoutedCallRequest() { // Flow Debugger (design 2026-09-05 §5.2): ExecuteAsync(trace: true) builds the script's // Route helper WithTrace(true), so a Route.To(...).Call(...) inside the script sends a // traced RouteToCallRequest; the default stays untraced. var locator = Substitute.For(); var router = Substitute.For(); locator.GetSiteIdForInstanceAsync("inst", Arg.Any()).Returns("SiteA"); var captured = new List(); router.RouteToCallAsync("SiteA", Arg.Do(captured.Add), Arg.Any()) .Returns(ci => new RouteToCallResponse(((RouteToCallRequest)ci[1]).CorrelationId, true, 1, null, DateTimeOffset.UtcNow)); var route = new RouteHelper(locator, router); var method = new ApiMethod("routed", "x") { Id = 9, TimeoutSeconds = 10 }; _executor.RegisterHandler("routed", ctx => ctx.Route.To("inst").Call("work")); var parent = Guid.NewGuid(); var traced = await _executor.ExecuteAsync(method, new Dictionary(), route, TimeSpan.FromSeconds(10), parentExecutionId: parent, trace: true); var plain = await _executor.ExecuteAsync(method, new Dictionary(), route, TimeSpan.FromSeconds(10)); Assert.True(traced.Success, traced.ErrorMessage); Assert.True(plain.Success, plain.ErrorMessage); Assert.Equal(2, captured.Count); Assert.True(captured[0].Trace); Assert.Equal(parent, captured[0].ParentExecutionId); Assert.False(captured[1].Trace); } [Fact] public async Task Executor_MethodWithSkipBodyCapture_RoutesWithBodyCaptureOff() { // Arch-review F11: a method configured with PerTargetOverrides[method].SkipBodyCapture // must build its Route helper WithBodyCapture(false), so the always-on RouteCall row // carries no request/response summary — mirroring AuditWriteMiddleware's own // SkipBodyCapture lookup for the inbound row (AuditWriteMiddleware.cs:183-187). var locator = Substitute.For(); var router = Substitute.For(); locator.GetSiteIdForInstanceAsync("inst", Arg.Any()).Returns("SiteA"); router.RouteToCallAsync("SiteA", Arg.Any(), Arg.Any()) .Returns(ci => new RouteToCallResponse(((RouteToCallRequest)ci[1]).CorrelationId, true, 1, null, DateTimeOffset.UtcNow)); var writer = Substitute.For(); AuditEvent? written = null; writer.WriteAsync(Arg.Do(e => written = e), Arg.Any()) .Returns(Task.CompletedTask); var route = new RouteHelper(locator, router, writer, null); var services = new ServiceCollection(); services.Configure(o => o.PerTargetOverrides["routed"] = new PerTargetRedactionOverride { SkipBodyCapture = true }); using var provider = services.BuildServiceProvider(); var executor = new InboundScriptExecutor(NullLogger.Instance, provider); var method = new ApiMethod("routed", "x") { Id = 9, TimeoutSeconds = 10 }; executor.RegisterHandler("routed", ctx => ctx.Route.To("inst").Call("work", new { x = 1 })); var result = await executor.ExecuteAsync(method, new Dictionary(), route, TimeSpan.FromSeconds(10)); Assert.True(result.Success, result.ErrorMessage); Assert.NotNull(written); var row = written!.AsRow(); Assert.Null(row.RequestSummary); Assert.Null(row.ResponseSummary); } [Fact] public async Task Executor_MethodWithoutSkipBodyCapture_RoutesWithBodyCaptureOn() { // Control: no PerTargetOverrides entry for this method → default behaviour unchanged. var locator = Substitute.For(); var router = Substitute.For(); locator.GetSiteIdForInstanceAsync("inst", Arg.Any()).Returns("SiteA"); router.RouteToCallAsync("SiteA", Arg.Any(), Arg.Any()) .Returns(ci => new RouteToCallResponse(((RouteToCallRequest)ci[1]).CorrelationId, true, 1, null, DateTimeOffset.UtcNow)); var writer = Substitute.For(); AuditEvent? written = null; writer.WriteAsync(Arg.Do(e => written = e), Arg.Any()) .Returns(Task.CompletedTask); var route = new RouteHelper(locator, router, writer, null); var services = new ServiceCollection(); services.Configure(o => o.PerTargetOverrides["other-method"] = new PerTargetRedactionOverride { SkipBodyCapture = true }); using var provider = services.BuildServiceProvider(); var executor = new InboundScriptExecutor(NullLogger.Instance, provider); var method = new ApiMethod("routed", "x") { Id = 9, TimeoutSeconds = 10 }; executor.RegisterHandler("routed", ctx => ctx.Route.To("inst").Call("work", new { x = 1 })); var result = await executor.ExecuteAsync(method, new Dictionary(), route, TimeSpan.FromSeconds(10)); Assert.True(result.Success, result.ErrorMessage); Assert.NotNull(written); var row = written!.AsRow(); Assert.Contains("\"x\":1", row.RequestSummary); Assert.Equal("1", row.ResponseSummary); } [Fact] public async Task RegisteredHandler_ExecutesSuccessfully() { var method = new ApiMethod("test", "return 42;") { Id = 1, TimeoutSeconds = 10 }; _executor.RegisterHandler("test", async ctx => { await Task.CompletedTask; return new { result = 42 }; }); var result = await _executor.ExecuteAsync( method, new Dictionary(), _route, TimeSpan.FromSeconds(10)); Assert.True(result.Success); Assert.NotNull(result.ResultJson); Assert.Contains("42", result.ResultJson); } [Fact] public async Task UnregisteredHandler_InvalidScript_ReturnsCompilationFailure() { // Use an invalid script that cannot be compiled by Roslyn var method = new ApiMethod("unknown", "%%% invalid C# %%%") { Id = 1, TimeoutSeconds = 10 }; var result = await _executor.ExecuteAsync( method, new Dictionary(), _route, TimeSpan.FromSeconds(10)); Assert.False(result.Success); Assert.Contains("Script compilation failed", result.ErrorMessage); } [Fact] public async Task UnregisteredHandler_ValidScript_LazyCompiles() { // Valid script that is not pre-registered triggers lazy compilation var method = new ApiMethod("lazy", "return 1;") { Id = 1, TimeoutSeconds = 10 }; var result = await _executor.ExecuteAsync( method, new Dictionary(), _route, TimeSpan.FromSeconds(10)); Assert.True(result.Success); } [Fact] public async Task HandlerThrows_ReturnsSafeErrorMessage() { var method = new ApiMethod("failing", "throw new Exception();") { Id = 1, TimeoutSeconds = 10 }; _executor.RegisterHandler("failing", _ => throw new InvalidOperationException("internal detail leak")); var result = await _executor.ExecuteAsync( method, new Dictionary(), _route, TimeSpan.FromSeconds(10)); Assert.False(result.Success); // WP-5: Safe error message — should NOT contain "internal detail leak" Assert.Equal("Internal script error", result.ErrorMessage); } [Fact] public async Task HandlerThrows_ErrorDetailCarriesTheException_WhileTheMessageStaysSafe() { // Flow Debugger step logs (design 2026-09-09 D3): the generic SCRIPT_ERROR catch keeps // the caller-facing message sanitised, but the exception itself must survive on the // additive ErrorDetail so the endpoint can put it on the inbound audit row (and the // Flow Debugger's inbound node can show why the run failed). var method = new ApiMethod("failing-detail", "throw new InvalidOperationException(\"real reason\");") { Id = 1, TimeoutSeconds = 10 }; _executor.RegisterHandler("failing-detail", _ => throw new InvalidOperationException("real reason")); var result = await _executor.ExecuteAsync( method, new Dictionary(), _route, TimeSpan.FromSeconds(10)); Assert.False(result.Success); Assert.Equal("Internal script error", result.ErrorMessage); Assert.Equal("SCRIPT_ERROR", result.ErrorCode); Assert.NotNull(result.ErrorDetail); Assert.Contains("InvalidOperationException", result.ErrorDetail); Assert.Contains("real reason", result.ErrorDetail); // The caller-safe fields must never carry it. Assert.DoesNotContain("real reason", result.ErrorMessage!); } [Fact] public async Task HandlerTimesOut_CarriesNoErrorDetail() { // A timeout is fully described by its message + TIMEOUT code — the // OperationCanceledException adds nothing, so the row stays detail-less. var method = new ApiMethod("slow-detail", "Thread.Sleep(60000);") { Id = 1, TimeoutSeconds = 1 }; _executor.RegisterHandler("slow-detail", async ctx => { await Task.Delay(TimeSpan.FromSeconds(60), ctx.CancellationToken); return "never"; }); var result = await _executor.ExecuteAsync( method, new Dictionary(), _route, TimeSpan.FromMilliseconds(100)); Assert.False(result.Success); Assert.Equal("TIMEOUT", result.ErrorCode); Assert.Null(result.ErrorDetail); } [Fact] public async Task RoutedAskTimeout_Returns500WithSiteUnreachableCode() { // N3: unreachability is the Ask EXPIRING (site never answered) — typed as // AskTimeoutException by the communication layer — not a substring in a // failure message. Pre-fix this fell into the generic catch as SCRIPT_ERROR. var locator = Substitute.For(); locator.GetSiteIdForInstanceAsync("X", Arg.Any()).Returns("SiteA"); var router = Substitute.For(); router.RouteToCallAsync("SiteA", Arg.Any(), Arg.Any()) .Returns(Task.FromException( new Akka.Actor.AskTimeoutException("Timeout after 00:00:30"))); var route = new RouteHelper(locator, router); var method = new ApiMethod("routed", "return await Route.To(\"X\").Call(\"s\");") { Id = 7, TimeoutSeconds = 10 }; var result = await _executor.ExecuteAsync( method, new Dictionary(), route, TimeSpan.FromSeconds(10)); Assert.False(result.Success); Assert.Equal("SITE_UNREACHABLE", result.ErrorCode); } [Fact] public async Task SiteRespondedFailure_MessageMentioningUnreachable_IsScriptErrorNotSiteUnreachable() { // N3 anti-sniff: a site that RESPONDED with Success=false is reachable by // definition — even when its error text happens to contain "unreachable". // Pre-fix the substring sniff misclassified this as SITE_UNREACHABLE. var locator = Substitute.For(); locator.GetSiteIdForInstanceAsync("X", Arg.Any()).Returns("SiteA"); var router = Substitute.For(); router.RouteToCallAsync("SiteA", Arg.Any(), Arg.Any()) .Returns(ci => new RouteToCallResponse( ((RouteToCallRequest)ci[1]).CorrelationId, Success: false, ReturnValue: null, ErrorMessage: "PLC device unreachable behind the site gateway", DateTimeOffset.UtcNow)); var route = new RouteHelper(locator, router); var method = new ApiMethod("routed2", "return await Route.To(\"X\").Call(\"s\");") { Id = 8, TimeoutSeconds = 10 }; var result = await _executor.ExecuteAsync( method, new Dictionary(), route, TimeSpan.FromSeconds(10)); Assert.False(result.Success); Assert.Equal("SCRIPT_ERROR", result.ErrorCode); } [Fact] public async Task HandlerTimesOut_ReturnsTimeoutError() { var method = new ApiMethod("slow", "Thread.Sleep(60000);") { Id = 1, TimeoutSeconds = 1 }; _executor.RegisterHandler("slow", async ctx => { await Task.Delay(TimeSpan.FromSeconds(60), ctx.CancellationToken); return "never"; }); var result = await _executor.ExecuteAsync( method, new Dictionary(), _route, TimeSpan.FromMilliseconds(100)); Assert.False(result.Success); Assert.Contains("timed out", result.ErrorMessage); } [Fact] public async Task HandlerAccessesParameters() { var method = new ApiMethod("echo", "return params;") { Id = 1, TimeoutSeconds = 10 }; _executor.RegisterHandler("echo", async ctx => { await Task.CompletedTask; return ctx.Parameters["name"]; }); var parameters = new Dictionary { { "name", "ScadaBridge" } }; var result = await _executor.ExecuteAsync( method, parameters, _route, TimeSpan.FromSeconds(10)); Assert.True(result.Success); Assert.Contains("ScadaBridge", result.ResultJson!); } // --- InboundAPI-001: concurrent lazy-compile must not corrupt the handler cache --- [Fact] public async Task ConcurrentLazyCompile_SameMethod_DoesNotCorruptCache() { // Many concurrent first-callers of an uncompiled method race the lazy-compile // path. With an unsynchronized Dictionary this can throw or return a torn/null // handler; all calls must succeed and produce the same result. var method = new ApiMethod("concurrent", "return 7;") { Id = 1, TimeoutSeconds = 10 }; var tasks = Enumerable.Range(0, 64).Select(_ => Task.Run(() => _executor.ExecuteAsync( method, new Dictionary(), _route, TimeSpan.FromSeconds(10)))).ToArray(); var results = await Task.WhenAll(tasks); Assert.All(results, r => { Assert.True(r.Success, r.ErrorMessage); Assert.Equal("7", r.ResultJson); }); } [Fact] public async Task ConcurrentRegisterAndExecute_DoesNotThrow() { // RegisterHandler/RemoveHandler racing ExecuteAsync must not crash the process // with an InvalidOperationException from concurrent Dictionary mutation. var method = new ApiMethod("racy", "return 1;") { Id = 1, TimeoutSeconds = 10 }; var writers = Enumerable.Range(0, 32).Select(i => Task.Run(() => { for (var n = 0; n < 50; n++) { _executor.RegisterHandler("racy", async ctx => { await Task.CompletedTask; return i; }); _executor.RemoveHandler("racy"); } })); var readers = Enumerable.Range(0, 32).Select(_ => Task.Run(async () => { for (var n = 0; n < 50; n++) { await _executor.ExecuteAsync( method, new Dictionary(), _route, TimeSpan.FromSeconds(10)); } })); // Should complete without an unhandled concurrency exception. await Task.WhenAll(writers.Concat(readers)); } // --- InboundAPI-005: compiled scripts must not bypass the script trust model --- [Theory] [InlineData("System.IO.File.Delete(\"/tmp/x\"); return null;")] [InlineData("System.Diagnostics.Process.Start(\"/bin/sh\"); return null;")] [InlineData("var t = System.Reflection.Assembly.GetExecutingAssembly(); return null;")] [InlineData("new System.Threading.Thread(() => {}).Start(); return null;")] [InlineData("var s = new System.Net.Sockets.Socket(System.Net.Sockets.AddressFamily.InterNetwork, System.Net.Sockets.SocketType.Stream, System.Net.Sockets.ProtocolType.Tcp); return null;")] public void CompileAndRegister_ForbiddenApi_RejectsScript(string script) { var method = new ApiMethod("forbidden", script) { Id = 1, TimeoutSeconds = 10 }; var registered = _executor.CompileAndRegister(method); Assert.False(registered); } [Fact] public async Task ExecuteAsync_ForbiddenApiScript_DoesNotRunAndReturnsFailure() { // A fully-qualified forbidden API call must be rejected at compile/register time // so the script never executes. var marker = System.IO.Path.Combine( System.IO.Path.GetTempPath(), $"scadabridge-pwned-{Guid.NewGuid():N}"); System.IO.File.Delete(marker); var method = new ApiMethod("evil", $"System.IO.File.WriteAllText(@\"{marker}\", \"x\"); return 1;") { Id = 1, TimeoutSeconds = 10 }; var result = await _executor.ExecuteAsync( method, new Dictionary(), _route, TimeSpan.FromSeconds(10)); Assert.False(result.Success); Assert.False(System.IO.File.Exists(marker)); } [Fact] public void CompileAndRegister_PermittedScript_StillRegisters() { // The trust-model check must not reject legitimate scripts. var method = new ApiMethod("ok", "var list = new List { 1, 2, 3 }; return list.Sum();") { Id = 1, TimeoutSeconds = 10 }; Assert.True(_executor.CompileAndRegister(method)); } // --- Compile-diagnostics overload: surface errors so a non-compiling save is not a silent no-op --- [Fact] public void CompileAndRegister_NonCompilingScript_ReportsErrors() { // A genuine Roslyn error (undefined symbol) must be reported via the out-param, // not just swallowed to a bare false. The Management Create/Update path turns // these into the caller-visible compileWarning. var method = new ApiMethod("broken", "return doesNotExist;") { Id = 1, TimeoutSeconds = 10 }; var ok = _executor.CompileAndRegister(method, out var errors); Assert.False(ok); Assert.NotEmpty(errors); } [Fact] public void CompileAndRegister_ValidScript_ReportsNoErrors() { var method = new ApiMethod("fine", "return 1;") { Id = 1, TimeoutSeconds = 10 }; var ok = _executor.CompileAndRegister(method, out var errors); Assert.True(ok); Assert.Empty(errors); } [Fact] public async Task CompileAndRegister_NonCompilingUpdate_ExecuteSurfacesCompileFailure() { // Arch-review S1 (DB-authoritative): CompileAndRegister still leaves the old // handler cached on a broken save (it never overwrites with a bad compile), but // ExecuteAsync now revision-checks the cached script against the freshly-fetched // row. A request carrying the broken row no longer silently serves the stale good // delegate — the mismatch triggers a recompile, which fails, so the method returns // a compilation error consistently on every node (honest failure over node-divergent // stale success). var good = new ApiMethod("evolving", "return 111;") { Id = 1, TimeoutSeconds = 10 }; Assert.True(_executor.CompileAndRegister(good, out var goodErrors)); Assert.Empty(goodErrors); var broken = new ApiMethod("evolving", "this is not valid C#") { Id = 1, TimeoutSeconds = 10 }; Assert.False(_executor.CompileAndRegister(broken, out var brokenErrors)); Assert.NotEmpty(brokenErrors); // The DB row is authoritative: the broken script now 500s rather than the old // delegate silently serving. var result = await _executor.ExecuteAsync( broken, new Dictionary(), _route, TimeSpan.FromSeconds(10)); Assert.False(result.Success); Assert.Contains("Script compilation failed", result.ErrorMessage); // The still-valid row keeps serving — only the broken revision fails. var stillGood = await _executor.ExecuteAsync( good, new Dictionary(), _route, TimeSpan.FromSeconds(10)); Assert.True(stillGood.Success); Assert.Contains("111", stillGood.ResultJson); } // --- InboundAPI-002: lazy compile-and-fetch must be atomic, never KeyNotFoundException --- [Fact] public async Task LazyCompile_RacingRemoveHandler_NeverThrowsKeyNotFound() { // The lazy-compile path must compile-and-fetch atomically: a concurrent // RemoveHandler must not be able to turn a first-call into an "Internal // script error" (the old check-then-act re-read could throw KeyNotFoundException). var method = new ApiMethod("atomic", "return 5;") { Id = 1, TimeoutSeconds = 10 }; var removers = Enumerable.Range(0, 16).Select(_ => Task.Run(() => { for (var n = 0; n < 200; n++) _executor.RemoveHandler("atomic"); })); var callers = Enumerable.Range(0, 16).Select(_ => Task.Run(async () => { for (var n = 0; n < 50; n++) { var r = await _executor.ExecuteAsync( method, new Dictionary(), _route, TimeSpan.FromSeconds(10)); // Result must always be a clean success or a clean compilation // failure — never the catch-all "Internal script error". Assert.NotEqual("Internal script error", r.ErrorMessage); } })); await Task.WhenAll(removers.Concat(callers)); } // --- InboundAPI-004: a client disconnect must NOT be reported as a script timeout --- [Fact] public async Task ClientDisconnect_IsNotReportedAsTimeout() { // When the caller's request token is cancelled (client aborted the request), // ExecuteAsync must report a client-cancelled failure, not "Script execution // timed out" — that log line is reserved for genuine timeouts. var method = new ApiMethod("aborted", "return 1;") { Id = 1, TimeoutSeconds = 30 }; _executor.RegisterHandler("aborted", async ctx => { await Task.Delay(TimeSpan.FromSeconds(60), ctx.CancellationToken); return "never"; }); using var clientAborted = new CancellationTokenSource(); clientAborted.CancelAfter(TimeSpan.FromMilliseconds(100)); var result = await _executor.ExecuteAsync( method, new Dictionary(), _route, // Generous method timeout so the timeout CTS is NOT the cause. TimeSpan.FromSeconds(30), clientAborted.Token); Assert.False(result.Success); Assert.DoesNotContain("timed out", result.ErrorMessage ?? string.Empty); } [Fact] public async Task GenuineTimeout_StillReportedAsTimeout() { // A method that exceeds its timeout with no client abort must still be // reported as "timed out" (regression guard for the InboundAPI-004 fix). var method = new ApiMethod("genuine", "return 1;") { Id = 1, TimeoutSeconds = 1 }; _executor.RegisterHandler("genuine", async ctx => { await Task.Delay(TimeSpan.FromSeconds(60), ctx.CancellationToken); return "never"; }); var result = await _executor.ExecuteAsync( method, new Dictionary(), _route, TimeSpan.FromMilliseconds(100), CancellationToken.None); Assert.False(result.Success); Assert.Contains("timed out", result.ErrorMessage); } // --- InboundAPI-009: a script that fails to compile must be compiled at most // once — repeated calls must not re-run the expensive Roslyn compilation. --- [Fact] public async Task FailedCompilation_IsNotRetriedOnEveryRequest() { // A broken script compiled once must be remembered as bad: subsequent // ExecuteAsync calls must NOT recompile (CPU amplification vector — there is // no rate limiting on the inbound API). Compilation is observed via the // "compilation failed" log line, which must appear exactly once. var counter = new CompileLogCounter(); var executor = new InboundScriptExecutor( new CountingLogger(counter), Substitute.For()); var method = new ApiMethod("broken", "%%% invalid C# %%%") { Id = 1, TimeoutSeconds = 10 }; for (var i = 0; i < 5; i++) { var result = await executor.ExecuteAsync( method, new Dictionary(), _route, TimeSpan.FromSeconds(10)); Assert.False(result.Success); } Assert.Equal(1, counter.CompilationFailures); } [Fact] public void FailedCompilation_RecompilesAfterCompileAndRegisterCalledAgain() { // The failure cache must not be permanent: when the method definition is // updated via CompileAndRegister, a now-valid script must register. var bad = new ApiMethod("fixable", "%%% invalid %%%") { Id = 1, TimeoutSeconds = 10 }; Assert.False(_executor.CompileAndRegister(bad)); var good = new ApiMethod("fixable", "return 1;") { Id = 1, TimeoutSeconds = 10 }; Assert.True(_executor.CompileAndRegister(good)); } // --- InboundAPI-024: _knownBadMethods must be bounded so a spam attack of // unique method names cannot grow the cache without bound. --- [Fact] public void KnownBadMethodsCache_SizeNeverExceedsCap_UnderUniqueNameFlood() { // Flood the executor with bad-method names well past the cache cap. The // cache must stabilise at or below the cap — any further unique bad name // is dropped rather than added (the per-request DB lookup remains the // correctness path; this cache is only a fast-fail optimisation). const int cap = 1000; const int floodCount = cap + 500; for (var i = 0; i < floodCount; i++) { var bad = new ApiMethod($"bad-{i}", "%%% invalid %%%") { Id = i + 1, TimeoutSeconds = 10 }; Assert.False(_executor.CompileAndRegister(bad)); } Assert.True( _executor.KnownBadMethodCount <= cap, $"known-bad cache size {_executor.KnownBadMethodCount} must not exceed cap {cap}"); } [Fact] public async Task KnownBadMethodsCache_LazyCompilePath_AlsoCappedUnderUniqueNameFlood() { // The lazy-compile path (ExecuteAsync on an unregistered method) records // failures via the same capped helper as CompileAndRegister, so flooding // it with unique URLs must not grow the cache without bound. const int cap = 1000; const int floodCount = cap + 250; for (var i = 0; i < floodCount; i++) { var method = new ApiMethod($"lazy-bad-{i}", "%%% invalid %%%") { Id = i + 1, TimeoutSeconds = 10 }; var result = await _executor.ExecuteAsync( method, new Dictionary(), _route, TimeSpan.FromSeconds(10)); Assert.False(result.Success); } Assert.True( _executor.KnownBadMethodCount <= cap, $"known-bad cache size {_executor.KnownBadMethodCount} must not exceed cap {cap}"); } // --- InboundAPI-014: the script return value is validated against ReturnDefinition --- [Fact] public async Task ReturnValue_MatchingReturnDefinition_Succeeds() { var method = new ApiMethod("shaped", "return x;") { Id = 1, TimeoutSeconds = 10, ReturnDefinition = """[{"name":"siteName","type":"String"},{"name":"total","type":"Integer"}]""", }; _executor.RegisterHandler("shaped", async ctx => { await Task.CompletedTask; return new { siteName = "Site Alpha", total = 14250 }; }); var result = await _executor.ExecuteAsync( method, new Dictionary(), _route, TimeSpan.FromSeconds(10)); Assert.True(result.Success, result.ErrorMessage); } [Fact] public async Task ReturnValue_NotMatchingReturnDefinition_ReturnsFailureNotMalformed200() { // The script returns a structure inconsistent with the declared return // definition (missing 'total'). It must surface as a failure, not a 200. var method = new ApiMethod("misshaped", "return x;") { Id = 1, TimeoutSeconds = 10, ReturnDefinition = """[{"name":"siteName","type":"String"},{"name":"total","type":"Integer"}]""", }; _executor.RegisterHandler("misshaped", async ctx => { await Task.CompletedTask; return new { siteName = "Site Alpha" }; // 'total' missing }); var result = await _executor.ExecuteAsync( method, new Dictionary(), _route, TimeSpan.FromSeconds(10)); Assert.False(result.Success); Assert.Null(result.ResultJson); } [Fact] public async Task ReturnValue_NoReturnDefinition_IsUnconstrained() { // A method with no ReturnDefinition keeps the prior behaviour — the return // value is serialized as-is. var method = new ApiMethod("free", "return x;") { Id = 1, TimeoutSeconds = 10 }; _executor.RegisterHandler("free", async ctx => { await Task.CompletedTask; return new { whatever = 1 }; }); var result = await _executor.ExecuteAsync( method, new Dictionary(), _route, TimeSpan.FromSeconds(10)); Assert.True(result.Success, result.ErrorMessage); Assert.Contains("whatever", result.ResultJson!); } // --- Audit Log #23 (ParentExecutionId, T3): the inbound request's // ExecutionId is threaded through ExecuteAsync onto routed calls --- [Fact] public async Task ExecuteAsync_WithParentExecutionId_RoutedCallCarriesItAsParentExecutionId() { // The endpoint hands ExecuteAsync the inbound request's ExecutionId; a // routed Route.To(...).Call(...) inside the script must stamp that id onto // the RouteToCallRequest as ParentExecutionId. var inboundExecutionId = Guid.NewGuid(); var locator = Substitute.For(); locator.GetSiteIdForInstanceAsync("inst-1", Arg.Any()).Returns("SiteA"); var router = Substitute.For(); RouteToCallRequest? captured = null; router.RouteToCallAsync("SiteA", Arg.Do(r => captured = r), Arg.Any()) .Returns(ci => new RouteToCallResponse( ((RouteToCallRequest)ci[1]).CorrelationId, true, null, null, DateTimeOffset.UtcNow)); var route = new RouteHelper(locator, router); var method = new ApiMethod("routes", "return 1;") { Id = 1, TimeoutSeconds = 10 }; _executor.RegisterHandler("routes", async ctx => { await ctx.Route.To("inst-1").Call("doWork"); return 1; }); var result = await _executor.ExecuteAsync( method, new Dictionary(), route, TimeSpan.FromSeconds(10), parentExecutionId: inboundExecutionId); Assert.True(result.Success, result.ErrorMessage); Assert.NotNull(captured); Assert.Equal(inboundExecutionId, captured!.ParentExecutionId); } [Fact] public async Task ExecuteAsync_WithoutParentExecutionId_RoutedCallHasNullParentExecutionId() { // ExecuteAsync called without a parent execution id (the default) routes // calls with ParentExecutionId null. var locator = Substitute.For(); locator.GetSiteIdForInstanceAsync("inst-1", Arg.Any()).Returns("SiteA"); var router = Substitute.For(); RouteToCallRequest? captured = null; router.RouteToCallAsync("SiteA", Arg.Do(r => captured = r), Arg.Any()) .Returns(ci => new RouteToCallResponse( ((RouteToCallRequest)ci[1]).CorrelationId, true, null, null, DateTimeOffset.UtcNow)); var route = new RouteHelper(locator, router); var method = new ApiMethod("routes2", "return 1;") { Id = 1, TimeoutSeconds = 10 }; _executor.RegisterHandler("routes2", async ctx => { await ctx.Route.To("inst-1").Call("doWork"); return 1; }); var result = await _executor.ExecuteAsync( method, new Dictionary(), route, TimeSpan.FromSeconds(10)); Assert.True(result.Success, result.ErrorMessage); Assert.NotNull(captured); Assert.Null(captured!.ParentExecutionId); } // --- IpsenMES MoveIn: the read-only Database helper is exposed on the script // context and resolves IDatabaseGateway from a DI scope per execution. --- [Fact] public async Task Script_UsingDatabase_QueriesViaGateway() { // A script that calls Database.QuerySingleAsync runs against an executor whose // ServiceProvider registers an IDatabaseGateway backed by in-memory SQLite. var services = new ServiceCollection(); services.AddSingleton(SeededSqliteGateway()); using var provider = services.BuildServiceProvider(); var executor = new InboundScriptExecutor( NullLogger.Instance, provider); var method = new ApiMethod( "movein", "return new { v = await Database.QuerySingleAsync(\"BTDB\", \"SELECT Code FROM Machine WHERE SAPID=@s\", new { s = (string)Parameters[\"sap\"] }) };") { Id = 1, TimeoutSeconds = 10, // null ReturnDefinition → ReturnValueValidator is unconstrained. ReturnDefinition = null, }; var result = await executor.ExecuteAsync( method, new Dictionary { { "sap", "131453" } }, _route, TimeSpan.FromSeconds(10)); Assert.True(result.Success, result.ErrorMessage); Assert.Contains("Z28061A", result.ResultJson!); } [Fact] public async Task Script_NotUsingDatabase_RunsWithProviderLackingGateway() { // A script that never touches Database must run even when no IDatabaseGateway // is registered — the helper is built via GetService (nullable) and only // throws on first use, which this script never reaches. var executor = new InboundScriptExecutor( NullLogger.Instance, Substitute.For()); var method = new ApiMethod("no-db", "return 99;") { Id = 1, TimeoutSeconds = 10 }; var result = await executor.ExecuteAsync( method, new Dictionary(), _route, TimeSpan.FromSeconds(10)); Assert.True(result.Success, result.ErrorMessage); Assert.Contains("99", result.ResultJson!); } [Fact] public async Task ScriptThrows_DisposesDiScope() { // When the script handler throws, ExecuteAsync must still dispose the // per-execution DI scope it created (regression guard for the finally block). var scopeServiceProvider = Substitute.For(); // GetService() returns null — script never needs it. scopeServiceProvider.GetService(typeof(IDatabaseGateway)).Returns((object?)null); var scope = Substitute.For(); scope.ServiceProvider.Returns(scopeServiceProvider); var factory = Substitute.For(); factory.CreateScope().Returns(scope); // Wire the factory into the provider so CreateScope() extension finds it. var provider = Substitute.For(); provider.GetService(typeof(IServiceScopeFactory)).Returns(factory); var executor = new InboundScriptExecutor( NullLogger.Instance, provider); // A pre-registered handler that throws; ReturnDefinition=null so validation // is not the failure path. var method = new ApiMethod("throws", "throw new Exception(\"boom\");") { Id = 1, TimeoutSeconds = 10, ReturnDefinition = null, }; executor.RegisterHandler("throws", _ => throw new Exception("boom")); var result = await executor.ExecuteAsync( method, new Dictionary(), _route, TimeSpan.FromSeconds(10)); // The executor must swallow the script exception and return a non-success result. Assert.False(result.Success); // The scope must have been disposed via the finally block. scope.Received(1).Dispose(); } // SQLite-backed IDatabaseGateway fake (mirrors InboundDatabaseHelperTests). The // shared in-memory db is seeded with Machine(Code,SAPID)=('Z28061A','131453'). private sealed class SqliteGateway : IDatabaseGateway { private readonly string _cs; public SqliteGateway(string cs) => _cs = cs; public async Task GetConnectionAsync(string name, CancellationToken ct = default) { var c = new SqliteConnection(_cs); await c.OpenAsync(ct); return c; } public Task CachedWriteAsync(string c, string s, IReadOnlyDictionary? p = null, string? o = null, CancellationToken ct = default, TrackedOperationId? t = null, Guid? e = null, string? src = null, Guid? pe = null) => throw new NotImplementedException(); } private static SqliteGateway SeededSqliteGateway() { var keep = new SqliteConnection("DataSource=file:movein-exec?mode=memory&cache=shared"); keep.Open(); // keep-alive: shared in-memory db lives until process exit using var cmd = keep.CreateCommand(); cmd.CommandText = "CREATE TABLE IF NOT EXISTS Machine(Code TEXT, SAPID TEXT); DELETE FROM Machine; INSERT INTO Machine VALUES('Z28061A','131453');"; cmd.ExecuteNonQuery(); return new SqliteGateway("DataSource=file:movein-exec?mode=memory&cache=shared"); } [Fact] public async Task TimedOutScript_ScopeDisposalDeferred_NoOrphanLeak() { // Handler ignores the token and keeps running past the method timeout. var release = new TaskCompletionSource(); _executor.RegisterHandler("slow", async _ => { await release.Task; return 1; }); var method = new ApiMethod("slow", "unused") { Id = 9, TimeoutSeconds = 1 }; var result = await _executor.ExecuteAsync( method, new Dictionary(), _route, TimeSpan.FromMilliseconds(50)); Assert.False(result.Success); Assert.Contains("timed out", result.ErrorMessage); Assert.Equal(1, _executor.AbandonedExecutionCount); // orphan is COUNTED release.SetResult(); // orphan finishes; continuation cleans up await Task.Delay(200); Assert.Equal(0, _executor.AbandonedExecutionCount); // ...and DECREMENTED } // --- Flow Debugger step logs (design 2026-09-09 D4/D5/D7): the Log global --- [Fact] public async Task Log_TracedRun_EmitsOneScriptLogRowOnTheTraceSink() { // D4/D5: a method script's Log.Info(...) binds as a top-level global and, because the // execution is traced, produces exactly one ScriptTrace/ScriptLog row carrying the // inbound request's own ExecutionId. stepEventId stays null on a central line — the // graph builder attaches it to the execution's anchor step (design D2). var sink = new List(); var route = NewRoute().WithTraceSink(sink.Add); var method = new ApiMethod("logs", "Log.Info(\"central\"); return 1;") { Id = 7, TimeoutSeconds = 10 }; var executionId = Guid.NewGuid(); var result = await _executor.ExecuteAsync( method, new Dictionary(), route, TimeSpan.FromSeconds(10), parentExecutionId: executionId, trace: true); Assert.True(result.Success, result.ErrorMessage); var row = Assert.Single(ScriptLogRows(sink)); var details = AuditDetailsCodec.Deserialize(row.DetailsJson); Assert.Equal(nameof(AuditKind.ScriptLog), details.Kind); Assert.Equal(nameof(AuditChannel.ScriptTrace), details.Channel); Assert.Equal(executionId, details.ExecutionId); Assert.Equal("logs", row.Target); Assert.Equal("central", details.RequestSummary); var extra = ScriptTraceExtra.TryParse(details.Extra); Assert.NotNull(extra); Assert.Equal("info", extra!.LogLevel); Assert.Equal("script", extra.LogSource); Assert.Null(extra.StepEventId); } [Fact] public async Task Log_TracedRun_WritesTheScriptLogRowToTheCentralAuditStore() { // D5: the row must be DURABLE, not just inline — Replay reads it back from the central // audit store, so the write happens even when no Live trace sink is attached, and it // carries the emitting node exactly as the RouteCall row does. var writer = Substitute.For(); writer.WriteAsync(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); var nodeIdentity = Substitute.For(); nodeIdentity.NodeName.Returns("central-a"); var services = new ServiceCollection(); services.AddSingleton(writer); services.AddSingleton(nodeIdentity); using var provider = services.BuildServiceProvider(); var executor = new InboundScriptExecutor(NullLogger.Instance, provider); var method = new ApiMethod("persisted", "Log.Info(\"persist\"); return 1;") { Id = 7, TimeoutSeconds = 10 }; var result = await executor.ExecuteAsync( method, new Dictionary(), NewRoute(), TimeSpan.FromSeconds(10), parentExecutionId: Guid.NewGuid(), trace: true); Assert.True(result.Success, result.ErrorMessage); await writer.Received(1).WriteAsync( Arg.Is(e => AuditDetailsCodec.Deserialize(e.DetailsJson).Kind == nameof(AuditKind.ScriptLog) && e.SourceNode == "central-a"), Arg.Any()); } [Fact] public async Task Log_UntracedRun_WritesOnlyToTheNodeLogger() { // D5: untraced runs pay nothing — no trace-sink row, no central audit write — but the // author's message still reaches the node's ILogger so it is never silently lost. var sink = new List(); var route = NewRoute().WithTraceSink(sink.Add); var writer = Substitute.For(); writer.WriteAsync(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); var services = new ServiceCollection(); services.AddSingleton(writer); using var provider = services.BuildServiceProvider(); var recorded = new List(); var executor = new InboundScriptExecutor(new RecordingLogger(recorded), provider); var method = new ApiMethod("quiet", "Log.Warn(\"unwatched\"); return 1;") { Id = 7, TimeoutSeconds = 10 }; var result = await executor.ExecuteAsync( method, new Dictionary(), route, TimeSpan.FromSeconds(10), parentExecutionId: Guid.NewGuid()); Assert.True(result.Success, result.ErrorMessage); Assert.Empty(ScriptLogRows(sink)); await writer.DidNotReceive().WriteAsync(Arg.Any(), Arg.Any()); Assert.Contains(recorded, m => m.Contains("unwatched", StringComparison.Ordinal)); } [Fact] public async Task Log_TracedWithException_CarriesMessageAndDetail() { // §4: the attached exception rides errorMessage/errorDetail, exactly as the failure // rows do, so the Log tab can render the type + stack. var sink = new List(); var route = NewRoute().WithTraceSink(sink.Add); var method = new ApiMethod( "boom", "Log.Warn(\"x\", new InvalidOperationException(\"e\")); return 1;") { Id = 7, TimeoutSeconds = 10 }; var result = await _executor.ExecuteAsync( method, new Dictionary(), route, TimeSpan.FromSeconds(10), parentExecutionId: Guid.NewGuid(), trace: true); Assert.True(result.Success, result.ErrorMessage); var details = AuditDetailsCodec.Deserialize(Assert.Single(ScriptLogRows(sink)).DetailsJson); Assert.Equal("x", details.RequestSummary); Assert.Equal("e", details.ErrorMessage); Assert.StartsWith("System.InvalidOperationException", details.ErrorDetail); Assert.Equal("warn", ScriptTraceExtra.TryParse(details.Extra)!.LogLevel); } [Fact] public async Task Log_PastMaxLogLinesPerRun_DropsAndEmitsOneClosingRuntimeWarning() { // D7: the per-run cap bounds the ROWS (the ILogger still sees every line); the run // ends with one runtime-sourced warn row naming how many lines were dropped. var sink = new List(); var route = NewRoute().WithTraceSink(sink.Add); var services = new ServiceCollection(); services.Configure(o => o.MaxLogLinesPerRun = 2); using var provider = services.BuildServiceProvider(); var executor = new InboundScriptExecutor(NullLogger.Instance, provider); var method = new ApiMethod( "chatty", "Log.Info(\"a\"); Log.Info(\"b\"); Log.Info(\"c\"); Log.Info(\"d\"); return 1;") { Id = 7, TimeoutSeconds = 10 }; var result = await executor.ExecuteAsync( method, new Dictionary(), route, TimeSpan.FromSeconds(10), parentExecutionId: Guid.NewGuid(), trace: true); Assert.True(result.Success, result.ErrorMessage); var rows = ScriptLogRows(sink) .Select(e => AuditDetailsCodec.Deserialize(e.DetailsJson)) .Select(d => (Details: d, Extra: ScriptTraceExtra.TryParse(d.Extra)!)) .ToList(); Assert.Equal(3, rows.Count); Assert.Equal(new[] { "a", "b" }, rows.Where(r => r.Extra.LogSource == "script").Select(r => r.Details.RequestSummary)); var closing = Assert.Single(rows, r => r.Extra.LogSource == "runtime"); Assert.Equal("warn", closing.Extra.LogLevel); Assert.Contains("2", closing.Details.RequestSummary); } private RouteHelper NewRoute() => new(Substitute.For(), Substitute.For()); private static List ScriptLogRows(IEnumerable events) => events.Where(e => AuditDetailsCodec.Deserialize(e.DetailsJson).Kind == nameof(AuditKind.ScriptLog)).ToList(); // Minimal ILogger that keeps every formatted message so a test can prove the node // logger saw a line even when nothing was audited. private sealed class RecordingLogger(List messages) : ILogger { public IDisposable? BeginScope(TState state) where TState : notnull => null; public bool IsEnabled(LogLevel logLevel) => true; public void Log( LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) { lock (messages) messages.Add(formatter(state, exception)); } } private sealed class CompileLogCounter { public int CompilationFailures; } private sealed class CountingLogger(CompileLogCounter counter) : ILogger { public IDisposable? BeginScope(TState state) where TState : notnull => null; public bool IsEnabled(LogLevel logLevel) => true; public void Log( LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) { var message = formatter(state, exception); if (message.Contains("script compilation failed", StringComparison.OrdinalIgnoreCase)) Interlocked.Increment(ref counter.CompilationFailures); } } }