using System.Text;
using System.Text.Json;
using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.TestHost;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NSubstitute;
using ScadaLink.AuditLog.Central;
using ScadaLink.AuditLog.Site;
using ScadaLink.AuditLog.Site.Telemetry;
using ScadaLink.AuditLog.Tests.Integration.Infrastructure;
using ScadaLink.Commons.Entities.Audit;
using ScadaLink.Commons.Interfaces.Repositories;
using ScadaLink.Commons.Interfaces.Services;
using ScadaLink.Commons.Messages.InboundApi;
using ScadaLink.Commons.Messages.Notification;
using ScadaLink.Commons.Types.Audit;
using ScadaLink.Commons.Types.Enums;
using ScadaLink.ConfigurationDatabase;
using ScadaLink.ConfigurationDatabase.Repositories;
using ScadaLink.ConfigurationDatabase.Tests.Migrations;
using ScadaLink.InboundAPI;
using ScadaLink.InboundAPI.Middleware;
using ScadaLink.NotificationOutbox;
using ScadaLink.NotificationOutbox.Delivery;
using ScadaLink.NotificationOutbox.Messages;
using ScadaLink.SiteRuntime.Scripts;
using ScadaLink.StoreAndForward;
namespace ScadaLink.AuditLog.Tests.Integration;
///
/// Audit Log #23 — ParentExecutionId cross-execution correlation headline
/// end-to-end suite. Verifies the inbound-API → routed-site-script bridge: an
/// inbound HTTP request runs an inbound method script that calls
/// Route.Call into a site instance; the routed site script does a sync
/// ExternalSystem.Call, a cached call and a Notify.Send. Every
/// audit row the routed run produces — site + central, sync + cached lifecycle
/// + NotifySend/NotifyDeliver — must carry
/// equal to the inbound request's
/// , while the routed run has its own
/// distinct and the inbound
/// row is top-level
/// (ParentExecutionId = NULL).
///
///
///
/// This is the integration-level counterpart to :
/// where that suite drives a single run and
/// asserts the shared per-run ExecutionId, this suite spans two
/// executions on opposite sides of the inbound→routed bridge and asserts the
/// cross-execution ParentExecutionId linkage plus
/// .
///
///
/// The bridge is exercised through the genuine production glue:
///
/// - the real in a
/// Microsoft.AspNetCore.TestHost pipeline — mints the inbound request's
/// per-request ExecutionId once, stashes it on
/// , and emits the top-level
/// row via the real
/// ;
/// - the real +
/// — the executor binds the stashed inbound
/// ExecutionId via , so a
/// Route.To(...).Call(...) inside the inbound script builds a
/// carrying
/// .
///
/// Only the cross-cluster routing transport is substituted: the test
/// stands in for
/// CommunicationServiceInstanceRouter exactly as the production site
/// (DeploymentManagerActor → ScriptActor → ScriptExecutionActor)
/// would — it reads off the
/// wire request and threads it into the routed
/// as parentExecutionId. A multi-node cluster is out of scope for an
/// in-process test (mirroring SiteAuditPushFlowTests's relay).
///
///
/// The central audit store is the real over the
/// per-class MSSQL database; the routed run's
/// site rows reach it through the real hot-path +
/// drain, the cached lifecycle rows through
/// the production , and the
/// NotifyDeliver rows through the real central
/// dispatcher.
///
///
public class ParentExecutionIdCorrelationTests : TestKit, IClassFixture
{
private readonly MsSqlMigrationFixture _fixture;
public ParentExecutionIdCorrelationTests(MsSqlMigrationFixture fixture)
{
_fixture = fixture;
}
private const string RoutedInstanceCode = "Plant.Pump42";
private const string RoutedScriptName = "OnInboundRouted";
private const string ExternalSystemName = "ERP";
private const string ExternalMethodName = "GetOrder";
private const string NotifyListName = "ops-team";
/// Per-run site id (Guid suffix) so concurrent tests sharing the MSSQL fixture stay isolated.
private static string NewSiteId() =>
"test-parentexec-" + Guid.NewGuid().ToString("N").Substring(0, 8);
private ScadaLinkDbContext CreateContext()
{
var options = new DbContextOptionsBuilder()
.UseSqlServer(_fixture.ConnectionString)
.ConfigureWarnings(w => w.Ignore(
Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning))
.Options;
return new ScadaLinkDbContext(options);
}
[SkippableFact]
public async Task InboundRoutedRun_AllRoutedRows_CarryInboundExecutionId_AsParentExecutionId()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var siteId = NewSiteId();
// ── Central — repository + ingest actor + audit writer over the MSSQL fixture ──
var centralServices = new ServiceCollection();
centralServices.AddDbContext(opts =>
opts.UseSqlServer(_fixture.ConnectionString)
.ConfigureWarnings(w => w.Ignore(
Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning)));
centralServices.AddScoped(sp =>
new AuditLogRepository(sp.GetRequiredService()));
centralServices.AddScoped(sp =>
new SiteCallAuditRepository(sp.GetRequiredService()));
centralServices.AddScoped(sp =>
new NotificationOutboxRepository(sp.GetRequiredService()));
centralServices.AddScoped(sp =>
new NotificationRepository(sp.GetRequiredService()));
// The NotifyDeliver dispatch path runs through this same long-lived
// provider — a stub adapter that always reports a successful delivery.
centralServices.AddScoped(_ => new AlwaysDeliversAdapter());
await using var centralProvider = centralServices.BuildServiceProvider();
var ingestActor = Sys.ActorOf(Props.Create(() => new AuditLogIngestActor(
(IServiceProvider)centralProvider,
NullLogger.Instance)));
var centralAuditWriter = new CentralAuditWriter(
centralProvider, NullLogger.Instance);
// ── Site — SQLite audit writer (hot-path) drained to central by the
// real SiteAuditTelemetryActor through the stub gRPC client. The sync
// ApiCall row and the NotifySend row flow through this chain. ──
await using var sqliteWriter = new SqliteAuditWriter(
Options.Create(new SqliteAuditWriterOptions
{
DatabasePath = "ignored",
BatchSize = 64,
ChannelCapacity = 1024,
}),
NullLogger.Instance,
connectionStringOverride:
$"Data Source=file:auditlog-parentexec-{Guid.NewGuid():N}?mode=memory&cache=shared");
var ring = new RingBufferFallback();
var siteAuditWriter = new FallbackAuditWriter(
sqliteWriter, ring, new NoOpAuditWriteFailureCounter(),
NullLogger.Instance);
var stubClient = new DirectActorSiteStreamAuditClient(ingestActor);
Sys.ActorOf(Props.Create(() => new SiteAuditTelemetryActor(
(ISiteAuditQueue)sqliteWriter,
stubClient,
Options.Create(new SiteAuditTelemetryOptions
{
BatchSize = 256,
BusyIntervalSeconds = 1,
IdleIntervalSeconds = 1,
}),
NullLogger.Instance)));
// Cached-call telemetry: production forwarder + dispatcher that also
// pushes each combined packet through the stub client into the central
// dual-write transaction (same wiring CombinedTelemetryHarness uses).
var cachedForwarder = new CombinedTelemetryDispatcher(
new CachedCallTelemetryForwarder(
siteAuditWriter, trackingStore: null,
NullLogger.Instance),
stubClient);
// Site Store-and-Forward — Notify.Send buffers a NotificationSubmit here.
using var safKeepAlive = new Microsoft.Data.Sqlite.SqliteConnection(
$"Data Source=parentexec-saf-{Guid.NewGuid():N};Mode=Memory;Cache=Shared");
safKeepAlive.Open();
var safStorage = new StoreAndForwardStorage(
safKeepAlive.ConnectionString, NullLogger.Instance);
await safStorage.InitializeAsync();
var storeAndForward = new StoreAndForwardService(
safStorage,
new StoreAndForwardOptions
{
DefaultRetryInterval = TimeSpan.Zero,
DefaultMaxRetries = 3,
RetryTimerInterval = TimeSpan.FromMinutes(10),
},
NullLogger.Instance);
// ── Outbound external-system client (routed run): sync Call succeeds,
// CachedCall completes immediately (WasBuffered=false) so the script
// helper emits the Submit + Attempted + CachedResolve lifecycle. ──
var externalClient = Substitute.For();
externalClient
.CallAsync(ExternalSystemName, ExternalMethodName,
Arg.Any?>(), Arg.Any())
.Returns(new ExternalCallResult(true, "{\"ok\":true}", null));
externalClient
.CachedCallAsync(ExternalSystemName, ExternalMethodName,
Arg.Any?>(),
Arg.Any(), Arg.Any(),
Arg.Any(),
Arg.Any(), Arg.Any(), Arg.Any())
.Returns(new ExternalCallResult(true, "{\"ok\":true}", null, WasBuffered: false));
// ── The routing transport stand-in: builds the routed ScriptRuntimeContext
// carrying RouteToCallRequest.ParentExecutionId — exactly what the
// production site handler (DeploymentManagerActor) does. ──
var router = new BridgingInstanceRouter(
siteId,
externalClient,
siteAuditWriter,
cachedForwarder,
storeAndForward);
// ── The inbound API method script: it calls Route.Call into the site
// instance. The real InboundScriptExecutor binds the inbound request's
// ExecutionId onto the RouteHelper, so the routed call carries it as
// ParentExecutionId. ──
var inboundMethod = new ScadaLink.Commons.Entities.InboundApi.ApiMethod(
"submitOrder",
$"return await Route.To(\"{RoutedInstanceCode}\").Call(\"{RoutedScriptName}\", new {{ order = 7 }});");
var locator = Substitute.For();
locator.GetSiteIdForInstanceAsync(RoutedInstanceCode, Arg.Any())
.Returns(siteId);
var scriptExecutor = new InboundScriptExecutor(
NullLogger.Instance,
new ServiceCollection().BuildServiceProvider());
Assert.True(scriptExecutor.CompileAndRegister(inboundMethod));
// ── Act — issue the inbound HTTP request through a TestHost pipeline
// fronted by the real AuditWriteMiddleware. The endpoint handler reads
// the middleware-stashed inbound ExecutionId and runs the inbound
// method script with it as parentExecutionId. ──
using var host = await BuildInboundHostAsync(centralAuditWriter, async ctx =>
{
var inboundExecutionId = (Guid)ctx.Items[AuditWriteMiddleware.InboundExecutionIdItemKey]!;
var route = new RouteHelper(locator, router);
var result = await scriptExecutor.ExecuteAsync(
inboundMethod,
new Dictionary(),
route,
TimeSpan.FromSeconds(30),
ctx.RequestAborted,
parentExecutionId: inboundExecutionId);
ctx.Response.StatusCode = result.Success ? 200 : 500;
await ctx.Response.WriteAsync(result.Success ? "ok" : "fail");
});
var client = host.GetTestClient();
var response = await client.PostAsync(
"/api/submitOrder",
new StringContent("{}", Encoding.UTF8, "application/json"));
Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode);
// The routed run produced a NotifySend that buffered a NotificationSubmit
// into S&F. Drain that genuine site-produced submission to the central
// NotificationOutboxActor so the NotifyDeliver dispatch rows materialise.
await ForwardBufferedNotificationToCentralAsync(
storeAndForward, router.NotificationId!, centralProvider, centralAuditWriter);
// ── Assert ──────────────────────────────────────────────────────────
await AwaitAssertAsync(async () =>
{
await using var readContext = CreateContext();
var repo = new AuditLogRepository(readContext);
// Every audit row this site produced (sync ApiCall + cached lifecycle
// + NotifySend) plus the central NotifyDeliver rows.
var siteRows = await repo.QueryAsync(
new AuditLogQueryFilter(SourceSiteIds: new[] { siteId }),
new AuditLogPaging(PageSize: 100));
// sync ApiCall (1) + cached Submit/Attempted/Resolve (3) + NotifySend (1)
// + NotifyDeliver Attempted/Delivered (2) = 7 rows for the routed run.
Assert.True(siteRows.Count == 7,
"Expected 7 routed-run audit rows; saw: "
+ string.Join(", ", siteRows.Select(r => $"{r.Channel}/{r.Kind}/{r.Status}")));
Assert.Single(siteRows, r => r.Channel == AuditChannel.ApiOutbound && r.Kind == AuditKind.ApiCall);
Assert.Single(siteRows, r => r.Kind == AuditKind.CachedSubmit);
Assert.Single(siteRows, r => r.Kind == AuditKind.CachedResolve);
Assert.Single(siteRows, r => r.Kind == AuditKind.NotifySend);
Assert.Equal(2, siteRows.Count(r => r.Kind == AuditKind.NotifyDeliver));
// CORE PROMISE: every routed-run row carries the SAME non-null
// ParentExecutionId — the inbound request's ExecutionId.
var parentIds = siteRows.Select(r => r.ParentExecutionId).Distinct().ToList();
Assert.Single(parentIds);
Assert.NotNull(parentIds[0]);
var inboundExecutionId = parentIds[0]!.Value;
// The routed run has its OWN distinct ExecutionId — not the parent's.
var routedExecutionIds = siteRows
.Select(r => r.ExecutionId)
.Distinct()
.ToList();
Assert.Single(routedExecutionIds);
Assert.NotNull(routedExecutionIds[0]);
var routedExecutionId = routedExecutionIds[0]!.Value;
Assert.NotEqual(inboundExecutionId, routedExecutionId);
// The inbound request's own InboundRequest row is TOP-LEVEL —
// ExecutionId = the propagated id, ParentExecutionId = NULL.
var inboundRows = await repo.QueryAsync(
new AuditLogQueryFilter(ExecutionId: inboundExecutionId),
new AuditLogPaging(PageSize: 10));
var inboundRow = Assert.Single(inboundRows,
r => r.Channel == AuditChannel.ApiInbound && r.Kind == AuditKind.InboundRequest);
Assert.Equal(AuditStatus.Delivered, inboundRow.Status);
Assert.Null(inboundRow.ParentExecutionId);
// The parentExecutionId filter pulls the routed run's complete
// trust-boundary footprint (all 7 routed rows, none of the inbound).
var byParent = await repo.QueryAsync(
new AuditLogQueryFilter(ParentExecutionId: inboundExecutionId),
new AuditLogPaging(PageSize: 100));
Assert.Equal(7, byParent.Count);
Assert.All(byParent, r => Assert.Equal(routedExecutionId, r.ExecutionId));
// GetExecutionTreeAsync returns BOTH executions in one chain —
// inbound (root) and routed (child), regardless of entry point.
var treeFromChild = await repo.GetExecutionTreeAsync(routedExecutionId);
AssertChain(treeFromChild, inboundExecutionId, routedExecutionId);
var treeFromRoot = await repo.GetExecutionTreeAsync(inboundExecutionId);
AssertChain(treeFromRoot, inboundExecutionId, routedExecutionId);
}, TimeSpan.FromSeconds(30));
}
///
/// Asserts the execution tree is the expected two-node inbound→routed chain:
/// the inbound execution is the root (ParentExecutionId = NULL) and the
/// routed execution's ParentExecutionId points back at it.
///
private static void AssertChain(
IReadOnlyList tree,
Guid inboundExecutionId,
Guid routedExecutionId)
{
Assert.Equal(2, tree.Count);
var root = Assert.Single(tree, n => n.ExecutionId == inboundExecutionId);
Assert.Null(root.ParentExecutionId);
var child = Assert.Single(tree, n => n.ExecutionId == routedExecutionId);
Assert.Equal(inboundExecutionId, child.ParentExecutionId);
}
///
/// Spins up a minimal in-memory ASP.NET host whose pipeline mirrors the
/// production inbound-API arrangement: routing → the real
/// → the POST /api/{methodName}
/// endpoint. The middleware mints + stashes the inbound request's
/// ExecutionId and emits the top-level
/// row via the supplied .
///
private static async Task BuildInboundHostAsync(
ICentralAuditWriter centralAuditWriter,
RequestDelegate endpointHandler)
{
var hostBuilder = new HostBuilder()
.ConfigureWebHost(webBuilder =>
{
webBuilder
.UseTestServer()
.ConfigureServices(services =>
{
services.AddSingleton(centralAuditWriter);
services.AddRouting();
})
.Configure(app =>
{
app.UseRouting();
app.UseAuditWriteMiddleware();
app.UseEndpoints(endpoints =>
{
endpoints.MapPost("/api/{methodName}", endpointHandler);
});
});
});
return await hostBuilder.StartAsync();
}
///
/// Reads the genuine site-produced the routed
/// Notify.Send buffered into Store-and-Forward, then drives it through
/// a real central so the
/// dispatch rows materialise. The
/// dispatcher echoes OriginParentExecutionId off the
/// NotificationSubmit onto every NotifyDeliver row — the
/// cross-execution linkage under test on the central side.
///
private async Task ForwardBufferedNotificationToCentralAsync(
StoreAndForwardService storeAndForward,
string notificationId,
IServiceProvider centralProvider,
ICentralAuditWriter centralAuditWriter)
{
var buffered = await storeAndForward.GetMessageByIdAsync(notificationId);
Assert.NotNull(buffered);
var submit = JsonSerializer.Deserialize(buffered!.PayloadJson);
Assert.NotNull(submit);
// The routed Notify.Send stamped the inbound request's ExecutionId as the
// submission's OriginParentExecutionId — proven separately on the
// NotifyDeliver rows, but asserted here too as the central handoff input.
Assert.NotNull(submit!.OriginParentExecutionId);
// The outbox actor runs over the long-lived central provider (which
// carries the AlwaysDeliversAdapter) so the dispatch sweep — launched
// asynchronously by the DispatchTick — still has a live IServiceProvider
// to resolve its per-sweep scope from.
var outboxActor = Sys.ActorOf(Props.Create(() => new NotificationOutboxActor(
centralProvider,
new NotificationOutboxOptions
{
// Long timers so PreStart's scheduled ticks never fire — the
// test drives ingest + dispatch explicitly.
DispatchInterval = TimeSpan.FromHours(1),
PurgeInterval = TimeSpan.FromDays(1),
},
centralAuditWriter,
NullLogger.Instance)));
// Ingest the genuine site submission, then run one dispatch sweep.
var ack = await outboxActor.Ask(
submit, TimeSpan.FromSeconds(15));
Assert.True(ack.Accepted, ack.Error);
outboxActor.Tell(InternalMessages.DispatchTick.Instance);
}
///
/// Stub that always reports a
/// successful delivery — a single dispatch sweep then yields one
/// + one
/// row.
///
private sealed class AlwaysDeliversAdapter : INotificationDeliveryAdapter
{
public NotificationType Type => NotificationType.Email;
public Task DeliverAsync(
ScadaLink.Commons.Entities.Notifications.Notification notification,
CancellationToken cancellationToken = default)
=> Task.FromResult(DeliveryOutcome.Success("ops@example.com"));
}
///
/// In-process stand-in for the cross-cluster routing transport
/// (CommunicationServiceInstanceRouter →
/// CommunicationService → site DeploymentManagerActor). On a
/// routed Call it does exactly what the production site handler does:
/// it reads off the wire
/// request and threads it into a fresh routed
/// as parentExecutionId, then runs the routed script's three
/// trust-boundary actions (sync ExternalSystem.Call, a cached call and
/// a Notify.Send). The routed context still mints its OWN fresh
/// ExecutionId — only the parent pointer is inherited.
///
private sealed class BridgingInstanceRouter : IInstanceRouter
{
private readonly string _siteId;
private readonly IExternalSystemClient _externalClient;
private readonly IAuditWriter _auditWriter;
private readonly ICachedCallTelemetryForwarder _cachedForwarder;
private readonly StoreAndForwardService _storeAndForward;
///
/// The NotificationId the routed Notify.Send minted, captured
/// so the test can drain the buffered .
///
public string? NotificationId { get; private set; }
public BridgingInstanceRouter(
string siteId,
IExternalSystemClient externalClient,
IAuditWriter auditWriter,
ICachedCallTelemetryForwarder cachedForwarder,
StoreAndForwardService storeAndForward)
{
_siteId = siteId;
_externalClient = externalClient;
_auditWriter = auditWriter;
_cachedForwarder = cachedForwarder;
_storeAndForward = storeAndForward;
}
public async Task RouteToCallAsync(
string siteId, RouteToCallRequest request, CancellationToken cancellationToken)
{
var compilationService = new ScriptCompilationService(
NullLogger.Instance);
var sharedScriptLibrary = new SharedScriptLibrary(
compilationService, NullLogger.Instance);
// Mirror DeploymentManagerActor → ScriptActor → ScriptExecutionActor:
// the routed script execution gets its OWN fresh ExecutionId, and the
// inbound request's ExecutionId arrives as ParentExecutionId.
var routedContext = new ScriptRuntimeContext(
ActorRefs.Nobody,
ActorRefs.Nobody,
sharedScriptLibrary,
currentCallDepth: 0,
maxCallDepth: 10,
askTimeout: TimeSpan.FromSeconds(5),
instanceName: request.InstanceUniqueName,
logger: NullLogger.Instance,
externalSystemClient: _externalClient,
databaseGateway: null,
storeAndForward: _storeAndForward,
siteCommunicationActor: null,
siteId: _siteId,
sourceScript: $"ScriptActor:{request.ScriptName}",
auditWriter: _auditWriter,
operationTrackingStore: null,
cachedForwarder: _cachedForwarder,
executionId: null,
parentExecutionId: request.ParentExecutionId);
// The routed site script's body: a sync ExternalSystem.Call, a cached
// call, and a Notify.Send — three distinct trust-boundary actions of
// the one routed execution.
await routedContext.ExternalSystem.Call(ExternalSystemName, ExternalMethodName);
await routedContext.ExternalSystem.CachedCall(ExternalSystemName, ExternalMethodName);
NotificationId = await routedContext.Notify
.To(NotifyListName)
.Send("Routed run alert", "inbound-routed script fired");
return new RouteToCallResponse(
request.CorrelationId, true, "routed-ok", null, DateTimeOffset.UtcNow);
}
public Task RouteToGetAttributesAsync(
string siteId, RouteToGetAttributesRequest request, CancellationToken cancellationToken)
=> throw new NotSupportedException();
public Task RouteToSetAttributesAsync(
string siteId, RouteToSetAttributesRequest request, CancellationToken cancellationToken)
=> throw new NotSupportedException();
}
}