using Microsoft.Extensions.Logging.Abstractions; using Moq; using ScadaLink.Commons.Entities.Audit; using ScadaLink.Commons.Interfaces.Services; using ScadaLink.Commons.Messages.Integration; using ScadaLink.Commons.Types; using ScadaLink.Commons.Types.Enums; using ScadaLink.SiteRuntime.Scripts; namespace ScadaLink.SiteRuntime.Tests.Scripts; /// /// Audit Log #23 — M3 Bundle E (Task E3): every script-initiated /// ExternalSystem.CachedCall emits exactly one CachedSubmit /// combined-telemetry packet at enqueue time, returns a fresh /// , and threads that id down to the /// store-and-forward layer so the retry-loop emissions (Tasks E4/E5) can join /// them by id. The audit emission is best-effort: a thrown forwarder must /// never abort the script's call, and the original /// must surface to the caller unchanged. /// public class ExternalSystemCachedCallEmissionTests { private sealed class CapturingForwarder : ICachedCallTelemetryForwarder { public List Telemetry { get; } = new(); public Exception? ThrowOnForward { get; set; } public Task ForwardAsync(CachedCallTelemetry telemetry, CancellationToken ct = default) { if (ThrowOnForward != null) { return Task.FromException(ThrowOnForward); } Telemetry.Add(telemetry); return Task.CompletedTask; } } private const string SiteId = "site-77"; private const string InstanceName = "Plant.Pump42"; private const string SourceScript = "ScriptActor:CheckPressure"; /// /// Audit Log #23: a fixed per-execution id so the cached-row tests can /// assert against a known value. /// private static readonly Guid TestExecutionId = Guid.NewGuid(); private static ScriptRuntimeContext.ExternalSystemHelper CreateHelper( IExternalSystemClient client, ICachedCallTelemetryForwarder? forwarder) { return new ScriptRuntimeContext.ExternalSystemHelper( client, InstanceName, NullLogger.Instance, // Audit Log #23: the per-execution id stamped into ExecutionId on // every script-side row. Cached rows keep CorrelationId = // TrackedOperationId (the per-operation lifecycle id). TestExecutionId, auditWriter: null, siteId: SiteId, sourceScript: SourceScript, cachedForwarder: forwarder); } [Fact] public async Task CachedCall_EmitsSubmitTelemetry_OnEnqueue() { var client = new Mock(); client .Setup(c => c.CachedCallAsync( "ERP", "GetOrder", It.IsAny?>(), InstanceName, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(new ExternalCallResult(true, null, null, WasBuffered: true)); var forwarder = new CapturingForwarder(); var helper = CreateHelper(client.Object, forwarder); var trackedId = await helper.CachedCall("ERP", "GetOrder"); Assert.NotEqual(default, trackedId); Assert.Single(forwarder.Telemetry); var packet = forwarder.Telemetry[0]; Assert.Equal(AuditChannel.ApiOutbound, packet.Audit.Channel); Assert.Equal(AuditKind.CachedSubmit, packet.Audit.Kind); Assert.Equal(AuditStatus.Submitted, packet.Audit.Status); Assert.Equal("ERP.GetOrder", packet.Audit.Target); // CorrelationId is the per-operation lifecycle id (TrackedOperationId); // ExecutionId is the per-execution id from the runtime context. Assert.Equal(trackedId.Value, packet.Audit.CorrelationId); Assert.Equal(TestExecutionId, packet.Audit.ExecutionId); Assert.Equal(AuditForwardState.Pending, packet.Audit.ForwardState); // Operational mirror — same id, Submitted, RetryCount 0, not terminal. Assert.Equal(trackedId, packet.Operational.TrackedOperationId); Assert.Equal("ApiOutbound", packet.Operational.Channel); Assert.Equal("ERP.GetOrder", packet.Operational.Target); Assert.Equal(SiteId, packet.Operational.SourceSite); Assert.Equal("Submitted", packet.Operational.Status); Assert.Equal(0, packet.Operational.RetryCount); Assert.Null(packet.Operational.LastError); Assert.Null(packet.Operational.TerminalAtUtc); } [Fact] public async Task CachedCall_ImmediateCompletion_CapturesRequestArgs_AndResponseBody() { var client = new Mock(); client .Setup(c => c.CachedCallAsync( "ERP", "GetOrder", It.IsAny?>(), InstanceName, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(new ExternalCallResult(true, "{\"ok\":true}", null, WasBuffered: false)); var forwarder = new CapturingForwarder(); var helper = CreateHelper(client.Object, forwarder); var args = new Dictionary { ["orderId"] = 42 }; await helper.CachedCall("ERP", "GetOrder", args); // Immediate completion (WasBuffered=false) emits Submit, Attempted, Resolve. Assert.Equal(3, forwarder.Telemetry.Count); var submit = forwarder.Telemetry.Single(t => t.Audit.Kind == AuditKind.CachedSubmit); var attempted = forwarder.Telemetry.Single(t => t.Audit.Kind == AuditKind.ApiCallCached); var resolve = forwarder.Telemetry.Single(t => t.Audit.Kind == AuditKind.CachedResolve); // Every row carries the request args; the two post-call rows also carry // the response body (Submit precedes the call, so it has no response). Assert.Equal("{\"orderId\":42}", submit.Audit.RequestSummary); Assert.Null(submit.Audit.ResponseSummary); Assert.Equal("{\"orderId\":42}", attempted.Audit.RequestSummary); Assert.Equal("{\"ok\":true}", attempted.Audit.ResponseSummary); Assert.Equal("{\"orderId\":42}", resolve.Audit.RequestSummary); Assert.Equal("{\"ok\":true}", resolve.Audit.ResponseSummary); } [Fact] public async Task CachedCall_ReturnsTrackedOperationId() { var client = new Mock(); client .Setup(c => c.CachedCallAsync( It.IsAny(), It.IsAny(), It.IsAny?>(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(new ExternalCallResult(true, null, null, WasBuffered: true)); var forwarder = new CapturingForwarder(); var helper = CreateHelper(client.Object, forwarder); var id1 = await helper.CachedCall("ERP", "GetOrder"); var id2 = await helper.CachedCall("ERP", "GetOrder"); Assert.NotEqual(default, id1); Assert.NotEqual(default, id2); Assert.NotEqual(id1, id2); // Both ids were threaded into the client invocations. client.Verify(c => c.CachedCallAsync( "ERP", "GetOrder", It.IsAny?>(), InstanceName, It.IsAny(), id1, It.IsAny(), It.IsAny()), Times.Once); client.Verify(c => c.CachedCallAsync( "ERP", "GetOrder", It.IsAny?>(), InstanceName, It.IsAny(), id2, It.IsAny(), It.IsAny()), Times.Once); } [Fact] public async Task CachedCall_ForwarderThrows_StillReturnsTrackedOperationId_OriginalCallProceeds() { var client = new Mock(); client .Setup(c => c.CachedCallAsync( It.IsAny(), It.IsAny(), It.IsAny?>(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(new ExternalCallResult(true, null, null, WasBuffered: true)); var forwarder = new CapturingForwarder { ThrowOnForward = new InvalidOperationException("simulated forwarder outage"), }; var helper = CreateHelper(client.Object, forwarder); // Must not throw — best-effort emission contract. var trackedId = await helper.CachedCall("ERP", "GetOrder"); Assert.NotEqual(default, trackedId); // The underlying call still ran exactly once. client.Verify(c => c.CachedCallAsync( "ERP", "GetOrder", It.IsAny?>(), InstanceName, It.IsAny(), trackedId, It.IsAny(), It.IsAny()), Times.Once); } [Fact] public async Task CachedCall_Provenance_Populated_FromContext() { var client = new Mock(); client .Setup(c => c.CachedCallAsync( It.IsAny(), It.IsAny(), It.IsAny?>(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(new ExternalCallResult(true, null, null, WasBuffered: true)); var forwarder = new CapturingForwarder(); var helper = CreateHelper(client.Object, forwarder); await helper.CachedCall("ERP", "GetOrder"); var packet = Assert.Single(forwarder.Telemetry); Assert.Equal(SiteId, packet.Audit.SourceSiteId); Assert.Equal(InstanceName, packet.Audit.SourceInstanceId); Assert.Equal(SourceScript, packet.Audit.SourceScript); Assert.Equal(SiteId, packet.Operational.SourceSite); } [Fact] public async Task CachedCall_NoForwarder_StillReturnsTrackedOperationId() { // Forwarder not wired (tests / minimal hosts) — must still return a // fresh id and invoke the underlying call. var client = new Mock(); client .Setup(c => c.CachedCallAsync( It.IsAny(), It.IsAny(), It.IsAny?>(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(new ExternalCallResult(true, null, null, WasBuffered: true)); var helper = CreateHelper(client.Object, forwarder: null); var trackedId = await helper.CachedCall("ERP", "GetOrder"); Assert.NotEqual(default, trackedId); client.Verify(c => c.CachedCallAsync( "ERP", "GetOrder", It.IsAny?>(), InstanceName, It.IsAny(), trackedId, It.IsAny(), It.IsAny()), Times.Once); } /// /// Audit Log #23 — M3 Bundle F (F2): when the underlying client call /// completes immediately (no S&F buffering, WasBuffered=false), /// the S&F retry loop never engages and the /// ICachedCallLifecycleObserver hook never fires. The cached-call /// helper itself must therefore emit the terminal lifecycle rows — /// otherwise Tracking.Status(id) would return Submitted /// forever and the audit log would be missing the Attempted / /// CachedResolve pair the M3 contract requires. /// /// Expected emissions on immediate success: /// 1. CachedSubmit / Submitted (already covered) /// 2. ApiCallCached / Attempted /// 3. CachedResolve / Delivered (TerminalAtUtc set) /// [Fact] public async Task CachedCall_ImmediateSuccess_EmitsAttemptedAndCachedResolve() { var client = new Mock(); client .Setup(c => c.CachedCallAsync( "ERP", "GetOrder", It.IsAny?>(), InstanceName, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) // WasBuffered=false — the immediate HTTP attempt succeeded; S&F // is bypassed entirely. .ReturnsAsync(new ExternalCallResult(true, "{\"ok\":true}", null, WasBuffered: false)); var forwarder = new CapturingForwarder(); var helper = CreateHelper(client.Object, forwarder); var trackedId = await helper.CachedCall("ERP", "GetOrder"); // Three telemetry packets emitted: Submit, Attempted, Resolve. Assert.Equal(3, forwarder.Telemetry.Count); var submit = forwarder.Telemetry[0]; Assert.Equal(AuditKind.CachedSubmit, submit.Audit.Kind); Assert.Equal(AuditStatus.Submitted, submit.Audit.Status); Assert.Equal(TestExecutionId, submit.Audit.ExecutionId); Assert.Equal(trackedId, submit.Operational.TrackedOperationId); Assert.Null(submit.Operational.TerminalAtUtc); var attempted = forwarder.Telemetry[1]; Assert.Equal(AuditChannel.ApiOutbound, attempted.Audit.Channel); Assert.Equal(AuditKind.ApiCallCached, attempted.Audit.Kind); Assert.Equal(AuditStatus.Attempted, attempted.Audit.Status); // Cached rows: CorrelationId = TrackedOperationId; ExecutionId is the // per-execution id from the runtime context. Assert.Equal(trackedId.Value, attempted.Audit.CorrelationId); Assert.Equal(TestExecutionId, attempted.Audit.ExecutionId); Assert.Equal("ERP.GetOrder", attempted.Audit.Target); Assert.Equal(trackedId, attempted.Operational.TrackedOperationId); Assert.Equal("Attempted", attempted.Operational.Status); Assert.Null(attempted.Operational.TerminalAtUtc); var resolve = forwarder.Telemetry[2]; Assert.Equal(AuditChannel.ApiOutbound, resolve.Audit.Channel); Assert.Equal(AuditKind.CachedResolve, resolve.Audit.Kind); Assert.Equal(AuditStatus.Delivered, resolve.Audit.Status); Assert.Equal(trackedId.Value, resolve.Audit.CorrelationId); Assert.Equal(TestExecutionId, resolve.Audit.ExecutionId); Assert.Equal(trackedId, resolve.Operational.TrackedOperationId); Assert.Equal("Delivered", resolve.Operational.Status); // Terminal row carries TerminalAtUtc. Assert.NotNull(resolve.Operational.TerminalAtUtc); } /// /// Audit Log #23 — M3 Bundle F (F2): the immediate-failure terminal /// path. When the client returns Success=false with /// WasBuffered=false (a permanent failure or a transient failure /// without an S&F engine to buffer it), the cached-call helper must /// still emit Attempted + CachedResolve with the failed status. /// [Fact] public async Task CachedCall_ImmediateFailure_EmitsAttemptedAndCachedResolveFailed() { var client = new Mock(); client .Setup(c => c.CachedCallAsync( "ERP", "GetOrder", It.IsAny?>(), InstanceName, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(new ExternalCallResult( false, null, "Permanent error: HTTP 422 bad payload", WasBuffered: false)); var forwarder = new CapturingForwarder(); var helper = CreateHelper(client.Object, forwarder); var trackedId = await helper.CachedCall("ERP", "GetOrder"); Assert.Equal(3, forwarder.Telemetry.Count); var attempted = forwarder.Telemetry[1]; Assert.Equal(AuditKind.ApiCallCached, attempted.Audit.Kind); Assert.Equal(AuditStatus.Attempted, attempted.Audit.Status); // The per-attempt row carries the error message. Assert.NotNull(attempted.Audit.ErrorMessage); var resolve = forwarder.Telemetry[2]; Assert.Equal(AuditKind.CachedResolve, resolve.Audit.Kind); // Immediate permanent failure -> Failed audit status / operational Failed. Assert.Equal(AuditStatus.Failed, resolve.Audit.Status); Assert.Equal("Failed", resolve.Operational.Status); Assert.NotNull(resolve.Operational.TerminalAtUtc); Assert.NotNull(resolve.Operational.LastError); } /// /// Audit Log #23 — M3 Bundle F (F2): when the client reports /// WasBuffered=true, the helper hands the operation to S&F and /// the retry-loop observer owns the Attempted + Resolve emissions. The /// helper must NOT emit those rows itself (otherwise we'd get duplicate /// Attempted + Resolve audit rows under the same tracking id). /// [Fact] public async Task CachedCall_BufferedPath_DoesNotEmitTerminalTelemetryFromHelper() { var client = new Mock(); client .Setup(c => c.CachedCallAsync( "ERP", "GetOrder", It.IsAny?>(), InstanceName, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) // S&F took ownership — Attempted + Resolve come from the // CachedCallLifecycleBridge driven by the retry loop, not the helper. .ReturnsAsync(new ExternalCallResult(true, null, null, WasBuffered: true)); var forwarder = new CapturingForwarder(); var helper = CreateHelper(client.Object, forwarder); await helper.CachedCall("ERP", "GetOrder"); // Only the CachedSubmit row — no Attempted / Resolve from the helper. var only = Assert.Single(forwarder.Telemetry); Assert.Equal(AuditKind.CachedSubmit, only.Audit.Kind); } }