Files
scadalink-design/tests/ScadaLink.SiteRuntime.Tests/Scripts/ExternalSystemCachedCallEmissionTests.cs

502 lines
22 KiB
C#

using Microsoft.Extensions.Logging.Abstractions;
using Moq;
using ScadaLink.Commons.Entities.Audit;
using ScadaLink.Commons.Interfaces.Services;
using ScadaLink.Commons.Messages.Integration;
using ScadaLink.Commons.Types;
using ScadaLink.Commons.Types.Enums;
using ScadaLink.SiteRuntime.Scripts;
namespace ScadaLink.SiteRuntime.Tests.Scripts;
/// <summary>
/// Audit Log #23 — M3 Bundle E (Task E3): every script-initiated
/// <c>ExternalSystem.CachedCall</c> emits exactly one <c>CachedSubmit</c>
/// combined-telemetry packet at enqueue time, returns a fresh
/// <see cref="TrackedOperationId"/>, and threads that id down to the
/// store-and-forward layer so the retry-loop emissions (Tasks E4/E5) can join
/// them by id. The audit emission is best-effort: a thrown forwarder must
/// never abort the script's call, and the original
/// <see cref="ExternalCallResult"/> must surface to the caller unchanged.
/// </summary>
public class ExternalSystemCachedCallEmissionTests
{
private sealed class CapturingForwarder : ICachedCallTelemetryForwarder
{
public List<CachedCallTelemetry> Telemetry { get; } = new();
public Exception? ThrowOnForward { get; set; }
public Task ForwardAsync(CachedCallTelemetry telemetry, CancellationToken ct = default)
{
if (ThrowOnForward != null)
{
return Task.FromException(ThrowOnForward);
}
Telemetry.Add(telemetry);
return Task.CompletedTask;
}
}
private const string SiteId = "site-77";
private const string InstanceName = "Plant.Pump42";
private const string SourceScript = "ScriptActor:CheckPressure";
/// <summary>
/// Audit Log #23: a fixed per-execution id so the cached-row tests can
/// assert <see cref="AuditEvent.ExecutionId"/> against a known value.
/// </summary>
private static readonly Guid TestExecutionId = Guid.NewGuid();
private static ScriptRuntimeContext.ExternalSystemHelper CreateHelper(
IExternalSystemClient client,
ICachedCallTelemetryForwarder? forwarder,
Guid? parentExecutionId = null)
{
return new ScriptRuntimeContext.ExternalSystemHelper(
client,
InstanceName,
NullLogger.Instance,
// Audit Log #23: the per-execution id stamped into ExecutionId on
// every script-side row. Cached rows keep CorrelationId =
// TrackedOperationId (the per-operation lifecycle id).
TestExecutionId,
auditWriter: null,
siteId: SiteId,
sourceScript: SourceScript,
cachedForwarder: forwarder,
parentExecutionId: parentExecutionId);
}
[Fact]
public async Task CachedCall_EmitsSubmitTelemetry_OnEnqueue()
{
var client = new Mock<IExternalSystemClient>();
client
.Setup(c => c.CachedCallAsync(
"ERP", "GetOrder",
It.IsAny<IReadOnlyDictionary<string, object?>?>(),
InstanceName,
It.IsAny<CancellationToken>(),
It.IsAny<TrackedOperationId?>(),
It.IsAny<Guid?>(), It.IsAny<string?>()))
.ReturnsAsync(new ExternalCallResult(true, null, null, WasBuffered: true));
var forwarder = new CapturingForwarder();
var helper = CreateHelper(client.Object, forwarder);
var trackedId = await helper.CachedCall("ERP", "GetOrder");
Assert.NotEqual(default, trackedId);
Assert.Single(forwarder.Telemetry);
var packet = forwarder.Telemetry[0];
Assert.Equal(AuditChannel.ApiOutbound, packet.Audit.Channel);
Assert.Equal(AuditKind.CachedSubmit, packet.Audit.Kind);
Assert.Equal(AuditStatus.Submitted, packet.Audit.Status);
Assert.Equal("ERP.GetOrder", packet.Audit.Target);
// CorrelationId is the per-operation lifecycle id (TrackedOperationId);
// ExecutionId is the per-execution id from the runtime context.
Assert.Equal(trackedId.Value, packet.Audit.CorrelationId);
Assert.Equal(TestExecutionId, packet.Audit.ExecutionId);
Assert.Equal(AuditForwardState.Pending, packet.Audit.ForwardState);
// Operational mirror — same id, Submitted, RetryCount 0, not terminal.
Assert.Equal(trackedId, packet.Operational.TrackedOperationId);
Assert.Equal("ApiOutbound", packet.Operational.Channel);
Assert.Equal("ERP.GetOrder", packet.Operational.Target);
Assert.Equal(SiteId, packet.Operational.SourceSite);
Assert.Equal("Submitted", packet.Operational.Status);
Assert.Equal(0, packet.Operational.RetryCount);
Assert.Null(packet.Operational.LastError);
Assert.Null(packet.Operational.TerminalAtUtc);
}
[Fact]
public async Task CachedCall_ImmediateCompletion_CapturesRequestArgs_AndResponseBody()
{
var client = new Mock<IExternalSystemClient>();
client
.Setup(c => c.CachedCallAsync(
"ERP", "GetOrder",
It.IsAny<IReadOnlyDictionary<string, object?>?>(),
InstanceName,
It.IsAny<CancellationToken>(),
It.IsAny<TrackedOperationId?>(),
It.IsAny<Guid?>(), It.IsAny<string?>()))
.ReturnsAsync(new ExternalCallResult(true, "{\"ok\":true}", null, WasBuffered: false));
var forwarder = new CapturingForwarder();
var helper = CreateHelper(client.Object, forwarder);
var args = new Dictionary<string, object?> { ["orderId"] = 42 };
await helper.CachedCall("ERP", "GetOrder", args);
// Immediate completion (WasBuffered=false) emits Submit, Attempted, Resolve.
Assert.Equal(3, forwarder.Telemetry.Count);
var submit = forwarder.Telemetry.Single(t => t.Audit.Kind == AuditKind.CachedSubmit);
var attempted = forwarder.Telemetry.Single(t => t.Audit.Kind == AuditKind.ApiCallCached);
var resolve = forwarder.Telemetry.Single(t => t.Audit.Kind == AuditKind.CachedResolve);
// Every row carries the request args; the two post-call rows also carry
// the response body (Submit precedes the call, so it has no response).
Assert.Equal("{\"orderId\":42}", submit.Audit.RequestSummary);
Assert.Null(submit.Audit.ResponseSummary);
Assert.Equal("{\"orderId\":42}", attempted.Audit.RequestSummary);
Assert.Equal("{\"ok\":true}", attempted.Audit.ResponseSummary);
Assert.Equal("{\"orderId\":42}", resolve.Audit.RequestSummary);
Assert.Equal("{\"ok\":true}", resolve.Audit.ResponseSummary);
}
[Fact]
public async Task CachedCall_ReturnsTrackedOperationId()
{
var client = new Mock<IExternalSystemClient>();
client
.Setup(c => c.CachedCallAsync(
It.IsAny<string>(), It.IsAny<string>(),
It.IsAny<IReadOnlyDictionary<string, object?>?>(),
It.IsAny<string?>(),
It.IsAny<CancellationToken>(),
It.IsAny<TrackedOperationId?>(),
It.IsAny<Guid?>(), It.IsAny<string?>()))
.ReturnsAsync(new ExternalCallResult(true, null, null, WasBuffered: true));
var forwarder = new CapturingForwarder();
var helper = CreateHelper(client.Object, forwarder);
var id1 = await helper.CachedCall("ERP", "GetOrder");
var id2 = await helper.CachedCall("ERP", "GetOrder");
Assert.NotEqual(default, id1);
Assert.NotEqual(default, id2);
Assert.NotEqual(id1, id2);
// Both ids were threaded into the client invocations.
client.Verify(c => c.CachedCallAsync(
"ERP", "GetOrder",
It.IsAny<IReadOnlyDictionary<string, object?>?>(),
InstanceName,
It.IsAny<CancellationToken>(),
id1,
It.IsAny<Guid?>(), It.IsAny<string?>()),
Times.Once);
client.Verify(c => c.CachedCallAsync(
"ERP", "GetOrder",
It.IsAny<IReadOnlyDictionary<string, object?>?>(),
InstanceName,
It.IsAny<CancellationToken>(),
id2,
It.IsAny<Guid?>(), It.IsAny<string?>()),
Times.Once);
}
/// <summary>
/// Audit Log #23 (ExecutionId Task 4): the helper → gateway hop of the
/// threading chain. The cached-call helper must forward the runtime
/// context's <c>ExecutionId</c> and <c>SourceScript</c> verbatim into
/// <see cref="IExternalSystemClient.CachedCallAsync"/> — so the buffered
/// retry loop later stamps the right provenance onto its audit rows.
/// This asserts the exact id/script (not <c>It.IsAny</c>), so a regression
/// that dropped the threading would fail here.
/// </summary>
[Fact]
public async Task CachedCall_ThreadsExecutionIdAndSourceScript_IntoClient()
{
var client = new Mock<IExternalSystemClient>();
client
.Setup(c => c.CachedCallAsync(
"ERP", "GetOrder",
It.IsAny<IReadOnlyDictionary<string, object?>?>(),
InstanceName,
It.IsAny<CancellationToken>(),
It.IsAny<TrackedOperationId?>(),
It.IsAny<Guid?>(), It.IsAny<string?>()))
.ReturnsAsync(new ExternalCallResult(true, null, null, WasBuffered: true));
var forwarder = new CapturingForwarder();
var helper = CreateHelper(client.Object, forwarder);
await helper.CachedCall("ERP", "GetOrder");
// The known TestExecutionId and SourceScript must reach the client
// unchanged — these are what the S&F retry loop persists and replays.
client.Verify(c => c.CachedCallAsync(
"ERP", "GetOrder",
It.IsAny<IReadOnlyDictionary<string, object?>?>(),
InstanceName,
It.IsAny<CancellationToken>(),
It.IsAny<TrackedOperationId?>(),
It.Is<Guid?>(id => id == TestExecutionId),
It.Is<string?>(s => s == SourceScript)),
Times.Once);
}
[Fact]
public async Task CachedCall_ForwarderThrows_StillReturnsTrackedOperationId_OriginalCallProceeds()
{
var client = new Mock<IExternalSystemClient>();
client
.Setup(c => c.CachedCallAsync(
It.IsAny<string>(), It.IsAny<string>(),
It.IsAny<IReadOnlyDictionary<string, object?>?>(),
It.IsAny<string?>(),
It.IsAny<CancellationToken>(),
It.IsAny<TrackedOperationId?>(),
It.IsAny<Guid?>(), It.IsAny<string?>()))
.ReturnsAsync(new ExternalCallResult(true, null, null, WasBuffered: true));
var forwarder = new CapturingForwarder
{
ThrowOnForward = new InvalidOperationException("simulated forwarder outage"),
};
var helper = CreateHelper(client.Object, forwarder);
// Must not throw — best-effort emission contract.
var trackedId = await helper.CachedCall("ERP", "GetOrder");
Assert.NotEqual(default, trackedId);
// The underlying call still ran exactly once.
client.Verify(c => c.CachedCallAsync(
"ERP", "GetOrder",
It.IsAny<IReadOnlyDictionary<string, object?>?>(),
InstanceName,
It.IsAny<CancellationToken>(),
trackedId,
It.IsAny<Guid?>(), It.IsAny<string?>()),
Times.Once);
}
[Fact]
public async Task CachedCall_Provenance_Populated_FromContext()
{
var client = new Mock<IExternalSystemClient>();
client
.Setup(c => c.CachedCallAsync(
It.IsAny<string>(), It.IsAny<string>(),
It.IsAny<IReadOnlyDictionary<string, object?>?>(),
It.IsAny<string?>(),
It.IsAny<CancellationToken>(),
It.IsAny<TrackedOperationId?>(),
It.IsAny<Guid?>(), It.IsAny<string?>()))
.ReturnsAsync(new ExternalCallResult(true, null, null, WasBuffered: true));
var forwarder = new CapturingForwarder();
var helper = CreateHelper(client.Object, forwarder);
await helper.CachedCall("ERP", "GetOrder");
var packet = Assert.Single(forwarder.Telemetry);
Assert.Equal(SiteId, packet.Audit.SourceSiteId);
Assert.Equal(InstanceName, packet.Audit.SourceInstanceId);
Assert.Equal(SourceScript, packet.Audit.SourceScript);
Assert.Equal(SiteId, packet.Operational.SourceSite);
}
[Fact]
public async Task CachedCall_NoForwarder_StillReturnsTrackedOperationId()
{
// Forwarder not wired (tests / minimal hosts) — must still return a
// fresh id and invoke the underlying call.
var client = new Mock<IExternalSystemClient>();
client
.Setup(c => c.CachedCallAsync(
It.IsAny<string>(), It.IsAny<string>(),
It.IsAny<IReadOnlyDictionary<string, object?>?>(),
It.IsAny<string?>(),
It.IsAny<CancellationToken>(),
It.IsAny<TrackedOperationId?>(),
It.IsAny<Guid?>(), It.IsAny<string?>()))
.ReturnsAsync(new ExternalCallResult(true, null, null, WasBuffered: true));
var helper = CreateHelper(client.Object, forwarder: null);
var trackedId = await helper.CachedCall("ERP", "GetOrder");
Assert.NotEqual(default, trackedId);
client.Verify(c => c.CachedCallAsync(
"ERP", "GetOrder",
It.IsAny<IReadOnlyDictionary<string, object?>?>(),
InstanceName,
It.IsAny<CancellationToken>(),
trackedId,
It.IsAny<Guid?>(), It.IsAny<string?>()),
Times.Once);
}
/// <summary>
/// Audit Log #23 — M3 Bundle F (F2): when the underlying client call
/// completes immediately (no S&amp;F buffering, <c>WasBuffered=false</c>),
/// the S&amp;F retry loop never engages and the
/// <c>ICachedCallLifecycleObserver</c> hook never fires. The cached-call
/// helper itself must therefore emit the terminal lifecycle rows —
/// otherwise <c>Tracking.Status(id)</c> would return <c>Submitted</c>
/// forever and the audit log would be missing the <c>Attempted</c> /
/// <c>CachedResolve</c> pair the M3 contract requires.
///
/// Expected emissions on immediate success:
/// 1. CachedSubmit / Submitted (already covered)
/// 2. ApiCallCached / Attempted
/// 3. CachedResolve / Delivered (TerminalAtUtc set)
/// </summary>
[Fact]
public async Task CachedCall_ImmediateSuccess_EmitsAttemptedAndCachedResolve()
{
var client = new Mock<IExternalSystemClient>();
client
.Setup(c => c.CachedCallAsync(
"ERP", "GetOrder",
It.IsAny<IReadOnlyDictionary<string, object?>?>(),
InstanceName,
It.IsAny<CancellationToken>(),
It.IsAny<TrackedOperationId?>(),
It.IsAny<Guid?>(), It.IsAny<string?>()))
// WasBuffered=false — the immediate HTTP attempt succeeded; S&F
// is bypassed entirely.
.ReturnsAsync(new ExternalCallResult(true, "{\"ok\":true}", null, WasBuffered: false));
var forwarder = new CapturingForwarder();
var helper = CreateHelper(client.Object, forwarder);
var trackedId = await helper.CachedCall("ERP", "GetOrder");
// Three telemetry packets emitted: Submit, Attempted, Resolve.
Assert.Equal(3, forwarder.Telemetry.Count);
var submit = forwarder.Telemetry[0];
Assert.Equal(AuditKind.CachedSubmit, submit.Audit.Kind);
Assert.Equal(AuditStatus.Submitted, submit.Audit.Status);
Assert.Equal(TestExecutionId, submit.Audit.ExecutionId);
Assert.Equal(trackedId, submit.Operational.TrackedOperationId);
Assert.Null(submit.Operational.TerminalAtUtc);
var attempted = forwarder.Telemetry[1];
Assert.Equal(AuditChannel.ApiOutbound, attempted.Audit.Channel);
Assert.Equal(AuditKind.ApiCallCached, attempted.Audit.Kind);
Assert.Equal(AuditStatus.Attempted, attempted.Audit.Status);
// Cached rows: CorrelationId = TrackedOperationId; ExecutionId is the
// per-execution id from the runtime context.
Assert.Equal(trackedId.Value, attempted.Audit.CorrelationId);
Assert.Equal(TestExecutionId, attempted.Audit.ExecutionId);
Assert.Equal("ERP.GetOrder", attempted.Audit.Target);
Assert.Equal(trackedId, attempted.Operational.TrackedOperationId);
Assert.Equal("Attempted", attempted.Operational.Status);
Assert.Null(attempted.Operational.TerminalAtUtc);
var resolve = forwarder.Telemetry[2];
Assert.Equal(AuditChannel.ApiOutbound, resolve.Audit.Channel);
Assert.Equal(AuditKind.CachedResolve, resolve.Audit.Kind);
Assert.Equal(AuditStatus.Delivered, resolve.Audit.Status);
Assert.Equal(trackedId.Value, resolve.Audit.CorrelationId);
Assert.Equal(TestExecutionId, resolve.Audit.ExecutionId);
Assert.Equal(trackedId, resolve.Operational.TrackedOperationId);
Assert.Equal("Delivered", resolve.Operational.Status);
// Terminal row carries TerminalAtUtc.
Assert.NotNull(resolve.Operational.TerminalAtUtc);
// Audit Log #23 (ParentExecutionId): null on every script-side cached
// row for a non-routed run.
Assert.Null(submit.Audit.ParentExecutionId);
Assert.Null(attempted.Audit.ParentExecutionId);
Assert.Null(resolve.Audit.ParentExecutionId);
}
[Fact]
public async Task CachedCall_RoutedRun_StampsParentExecutionId_OnAllScriptSideRows()
{
// Audit Log #23 (ParentExecutionId, Task 5): an inbound-API-routed run
// carries the spawning execution's id; every script-side cached row
// (CachedSubmit, ApiCallCached, CachedResolve) must stamp it in
// ParentExecutionId.
var parentExecutionId = Guid.NewGuid();
var client = new Mock<IExternalSystemClient>();
client
.Setup(c => c.CachedCallAsync(
"ERP", "GetOrder",
It.IsAny<IReadOnlyDictionary<string, object?>?>(),
InstanceName,
It.IsAny<CancellationToken>(),
It.IsAny<TrackedOperationId?>(),
It.IsAny<Guid?>(), It.IsAny<string?>()))
.ReturnsAsync(new ExternalCallResult(true, "{\"ok\":true}", null, WasBuffered: false));
var forwarder = new CapturingForwarder();
var helper = CreateHelper(client.Object, forwarder, parentExecutionId);
await helper.CachedCall("ERP", "GetOrder");
Assert.Equal(3, forwarder.Telemetry.Count);
Assert.All(forwarder.Telemetry, t =>
Assert.Equal(parentExecutionId, t.Audit.ParentExecutionId));
}
/// <summary>
/// Audit Log #23 — M3 Bundle F (F2): the immediate-failure terminal
/// path. When the client returns <c>Success=false</c> with
/// <c>WasBuffered=false</c> (a permanent failure or a transient failure
/// without an S&amp;F engine to buffer it), the cached-call helper must
/// still emit Attempted + CachedResolve with the failed status.
/// </summary>
[Fact]
public async Task CachedCall_ImmediateFailure_EmitsAttemptedAndCachedResolveFailed()
{
var client = new Mock<IExternalSystemClient>();
client
.Setup(c => c.CachedCallAsync(
"ERP", "GetOrder",
It.IsAny<IReadOnlyDictionary<string, object?>?>(),
InstanceName,
It.IsAny<CancellationToken>(),
It.IsAny<TrackedOperationId?>(),
It.IsAny<Guid?>(), It.IsAny<string?>()))
.ReturnsAsync(new ExternalCallResult(
false, null, "Permanent error: HTTP 422 bad payload", WasBuffered: false));
var forwarder = new CapturingForwarder();
var helper = CreateHelper(client.Object, forwarder);
var trackedId = await helper.CachedCall("ERP", "GetOrder");
Assert.Equal(3, forwarder.Telemetry.Count);
var attempted = forwarder.Telemetry[1];
Assert.Equal(AuditKind.ApiCallCached, attempted.Audit.Kind);
Assert.Equal(AuditStatus.Attempted, attempted.Audit.Status);
// The per-attempt row carries the error message.
Assert.NotNull(attempted.Audit.ErrorMessage);
var resolve = forwarder.Telemetry[2];
Assert.Equal(AuditKind.CachedResolve, resolve.Audit.Kind);
// Immediate permanent failure -> Failed audit status / operational Failed.
Assert.Equal(AuditStatus.Failed, resolve.Audit.Status);
Assert.Equal("Failed", resolve.Operational.Status);
Assert.NotNull(resolve.Operational.TerminalAtUtc);
Assert.NotNull(resolve.Operational.LastError);
}
/// <summary>
/// Audit Log #23 — M3 Bundle F (F2): when the client reports
/// <c>WasBuffered=true</c>, the helper hands the operation to S&amp;F and
/// the retry-loop observer owns the Attempted + Resolve emissions. The
/// helper must NOT emit those rows itself (otherwise we'd get duplicate
/// Attempted + Resolve audit rows under the same tracking id).
/// </summary>
[Fact]
public async Task CachedCall_BufferedPath_DoesNotEmitTerminalTelemetryFromHelper()
{
var client = new Mock<IExternalSystemClient>();
client
.Setup(c => c.CachedCallAsync(
"ERP", "GetOrder",
It.IsAny<IReadOnlyDictionary<string, object?>?>(),
InstanceName,
It.IsAny<CancellationToken>(),
It.IsAny<TrackedOperationId?>(),
It.IsAny<Guid?>(), It.IsAny<string?>()))
// S&F took ownership — Attempted + Resolve come from the
// CachedCallLifecycleBridge driven by the retry loop, not the helper.
.ReturnsAsync(new ExternalCallResult(true, null, null, WasBuffered: true));
var forwarder = new CapturingForwarder();
var helper = CreateHelper(client.Object, forwarder);
await helper.CachedCall("ERP", "GetOrder");
// Only the CachedSubmit row — no Attempted / Resolve from the helper.
var only = Assert.Single(forwarder.Telemetry);
Assert.Equal(AuditKind.CachedSubmit, only.Audit.Kind);
}
}