feat(sitecall-audit): carry + persist SourceNode end-to-end via cached telemetry

Site: site emitters of SiteCallOperational (ExternalSystemClient, the script-API
cached call path in ScriptRuntimeContext, CachedCallLifecycleBridge) inject
INodeIdentityProvider and stamp SourceNode = NodeName at construction.

OperationTrackingStore call site in CachedCallTelemetryForwarder now stamps
SourceNode too.

Central: SiteCallAuditRepository.UpsertAsync INSERT includes SourceNode in the
column list; conditional monotonic UPDATE uses
COALESCE(@SourceNode, SourceNode) so later packets cannot blank a previously-
stamped value. After this commit every SiteCalls row carries node-a/node-b in
SourceNode (subject to monotonic preservation).
This commit is contained in:
Joseph Doherty
2026-05-23 17:41:22 -04:00
parent d1fcab490c
commit 06ed0acead
10 changed files with 539 additions and 34 deletions

View File

@@ -327,4 +327,68 @@ public class CachedCallLifecycleBridgeTests
Assert.NotNull(captured);
Assert.Null(captured!.Audit.ParentExecutionId);
}
// ── SourceNode-stamping (Task 14) ──
[Fact]
public async Task RetryLoopRow_StampsSourceNode_FromNodeIdentityProvider()
{
// SourceNode-stamping (Task 14): when an INodeIdentityProvider is
// wired the bridge stamps the local node name (node-a/node-b) onto
// the SiteCallOperational.SourceNode column of every emitted packet.
var nodeIdentity = Substitute.For<INodeIdentityProvider>();
nodeIdentity.NodeName.Returns("node-a");
var captured = new List<CachedCallTelemetry>();
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured.Add(t)), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
var sut = new CachedCallLifecycleBridge(
_forwarder, NullLogger<CachedCallLifecycleBridge>.Instance, nodeIdentity);
await sut.OnAttemptCompletedAsync(Ctx(CachedCallAttemptOutcome.Delivered));
Assert.Equal(2, captured.Count);
Assert.All(captured, p => Assert.Equal("node-a", p.Operational.SourceNode));
}
[Fact]
public async Task RetryLoopRow_NoNodeIdentityProvider_LeavesSourceNodeNull()
{
// When no INodeIdentityProvider is wired (legacy hosts / tests) the
// bridge degrades to a null SourceNode rather than throwing. The
// emitted packet's SourceNode is null so the central row persists NULL.
var captured = new List<CachedCallTelemetry>();
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured.Add(t)), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
// Default CreateSut() does NOT pass a node-identity provider.
var sut = CreateSut();
await sut.OnAttemptCompletedAsync(Ctx(CachedCallAttemptOutcome.TransientFailure));
var packet = Assert.Single(captured);
Assert.Null(packet.Operational.SourceNode);
}
[Fact]
public async Task RetryLoopRow_NodeIdentityWithNullNodeName_LeavesSourceNodeNull()
{
// The provider exists but reports a null NodeName (unconfigured). The
// bridge must pass that null through to SourceNode rather than
// falling back to a placeholder.
var nodeIdentity = Substitute.For<INodeIdentityProvider>();
nodeIdentity.NodeName.Returns((string?)null);
var captured = new List<CachedCallTelemetry>();
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured.Add(t)), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
var sut = new CachedCallLifecycleBridge(
_forwarder, NullLogger<CachedCallLifecycleBridge>.Instance, nodeIdentity);
await sut.OnAttemptCompletedAsync(Ctx(CachedCallAttemptOutcome.TransientFailure));
var packet = Assert.Single(captured);
Assert.Null(packet.Operational.SourceNode);
}
}

View File

@@ -135,7 +135,11 @@ public class CachedCallTelemetryForwarderTests
Arg.Any<CancellationToken>());
// Tracking row: insert-if-not-exists with kind discriminator.
// sourceNode is null until Task 14 wires the INodeIdentityProvider through.
// Default CreateSut() does NOT supply an INodeIdentityProvider, so the
// forwarder passes null sourceNode to RecordEnqueueAsync (legacy / test
// host behaviour). The Task 14 stamping path is covered by the
// ForwardAsync_Submit_StampsSourceNode_FromNodeIdentityProvider test
// below.
await _tracking.Received(1).RecordEnqueueAsync(
_id,
"ApiOutbound",
@@ -249,4 +253,55 @@ public class CachedCallTelemetryForwarderTests
await Assert.ThrowsAsync<ArgumentNullException>(
() => sut.ForwardAsync(null!, CancellationToken.None));
}
// ── SourceNode-stamping (Task 14) ──
[Fact]
public async Task ForwardAsync_Submit_StampsSourceNode_FromNodeIdentityProvider()
{
// SourceNode-stamping (Task 14): when an INodeIdentityProvider is
// wired the forwarder must stamp its NodeName onto the
// RecordEnqueueAsync sourceNode parameter so the tracking row
// captures the originating node (node-a/node-b).
var nodeIdentity = Substitute.For<INodeIdentityProvider>();
nodeIdentity.NodeName.Returns("node-a");
var sut = new CachedCallTelemetryForwarder(
_writer, _tracking, NullLogger<CachedCallTelemetryForwarder>.Instance, nodeIdentity);
await sut.ForwardAsync(SubmitPacket(), CancellationToken.None);
await _tracking.Received(1).RecordEnqueueAsync(
_id,
"ApiOutbound",
"ERP.GetOrder",
"inst-1",
"ScriptActor:doStuff",
"node-a",
Arg.Any<CancellationToken>());
}
[Fact]
public async Task ForwardAsync_Submit_NodeIdentityNullNodeName_PassesNullSourceNode()
{
// The provider exists but reports a null NodeName (unconfigured).
// The forwarder passes that null through to RecordEnqueueAsync rather
// than falling back to a placeholder string.
var nodeIdentity = Substitute.For<INodeIdentityProvider>();
nodeIdentity.NodeName.Returns((string?)null);
var sut = new CachedCallTelemetryForwarder(
_writer, _tracking, NullLogger<CachedCallTelemetryForwarder>.Instance, nodeIdentity);
await sut.ForwardAsync(SubmitPacket(), CancellationToken.None);
await _tracking.Received(1).RecordEnqueueAsync(
_id,
"ApiOutbound",
"ERP.GetOrder",
"inst-1",
"ScriptActor:doStuff",
null,
Arg.Any<CancellationToken>());
}
}

View File

@@ -520,7 +520,8 @@ public class SiteCallAuditRepositoryTests : IClassFixture<MsSqlMigrationFixture>
DateTime? createdAtUtc = null,
DateTime? updatedAtUtc = null,
bool terminal = false,
DateTime? terminalAtUtc = null)
DateTime? terminalAtUtc = null,
string? sourceNode = null)
{
var created = createdAtUtc ?? DateTime.UtcNow;
var updated = updatedAtUtc ?? created;
@@ -534,6 +535,7 @@ public class SiteCallAuditRepositoryTests : IClassFixture<MsSqlMigrationFixture>
Channel = "ApiOutbound",
Target = "ERP.GetOrder",
SourceSite = sourceSite ?? NewSiteId(),
SourceNode = sourceNode,
Status = status,
RetryCount = retryCount,
LastError = lastError,
@@ -544,4 +546,159 @@ public class SiteCallAuditRepositoryTests : IClassFixture<MsSqlMigrationFixture>
IngestedAtUtc = DateTime.UtcNow,
};
}
// --- SourceNode-stamping (Task 14) --------------------------------------
[SkippableFact]
public async Task UpsertAsync_PersistsSourceNode_OnFreshInsert()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
// SourceNode-stamping (Task 14): a fresh INSERT must persist the
// SourceNode column verbatim — the central row carries the originating
// site node name end-to-end.
var id = TrackedOperationId.New();
await using var context = CreateContext();
var repo = new SiteCallAuditRepository(context);
await repo.UpsertAsync(NewRow(id, status: "Submitted", sourceNode: "node-a"));
var loaded = await repo.GetAsync(id);
Assert.NotNull(loaded);
Assert.Equal("node-a", loaded!.SourceNode);
}
[SkippableFact]
public async Task UpsertAsync_PreservesSourceNode_WhenLaterPacketCarriesNull()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
// SourceNode-stamping (Task 14): the UPDATE uses
// COALESCE(@SourceNode, SourceNode) so a subsequent packet that does
// NOT carry a SourceNode (legacy / reconciliation pull from an
// unstamped node) MUST NOT blank out the value the first packet set.
// Combined with the monotonic-rank guard the Status advances but the
// SourceNode survives.
//
// Each step uses a fresh DbContext — raw-SQL UPDATEs bypass the
// change tracker, so reusing a single context whose entity is already
// tracked masks the post-UPDATE state on a follow-up FindAsync.
var id = TrackedOperationId.New();
await using (var context = CreateContext())
{
var repo = new SiteCallAuditRepository(context);
// First packet: stamped Submit from node-a.
await repo.UpsertAsync(NewRow(id, status: "Submitted", sourceNode: "node-a"));
}
await using (var context = CreateContext())
{
var repo = new SiteCallAuditRepository(context);
// Later packet: rank-advancing Attempted with null SourceNode.
await repo.UpsertAsync(NewRow(id, status: "Attempted", retryCount: 1, sourceNode: null));
}
await using (var readContext = CreateContext())
{
var readRepo = new SiteCallAuditRepository(readContext);
var loaded = await readRepo.GetAsync(id);
Assert.NotNull(loaded);
// SourceNode preserved despite the null on the later packet.
Assert.Equal("node-a", loaded!.SourceNode);
// Status advanced — proves the UPDATE branch actually ran.
Assert.Equal("Attempted", loaded.Status);
Assert.Equal(1, loaded.RetryCount);
}
}
[SkippableFact]
public async Task UpsertAsync_NonNullIncomingSourceNode_OverwritesPreviousValueOnRankAdvance()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
// SourceNode-stamping (Task 14): per the COALESCE(@SourceNode,
// SourceNode) semantics the column protects against a *null*
// incoming value blanking a previously-stamped one, but a non-null
// incoming value DOES replace the existing value on a rank-advancing
// packet. This is the "last-non-null-wins on advance" behaviour the
// SQL operator literally implements — see the comment in
// SiteCallAuditRepository.UpsertAsync.
//
// In practice both stamps within a single lifecycle SHOULD carry the
// same value (same node, same execution); a divergence would imply a
// mid-lifecycle node change (e.g. failover handing off to node-b) and
// letting the latest stamp through is arguably the right call. This
// test pins the actual behaviour so we notice if the SQL gets
// inverted (to a true first-write-wins COALESCE(SourceNode,
// @SourceNode)) inadvertently.
var id = TrackedOperationId.New();
await using (var context = CreateContext())
{
var repo = new SiteCallAuditRepository(context);
await repo.UpsertAsync(NewRow(id, status: "Submitted", sourceNode: "node-a"));
}
await using (var context = CreateContext())
{
var repo = new SiteCallAuditRepository(context);
await repo.UpsertAsync(NewRow(id, status: "Attempted", retryCount: 1, sourceNode: "node-b"));
}
await using (var readContext = CreateContext())
{
var readRepo = new SiteCallAuditRepository(readContext);
var loaded = await readRepo.GetAsync(id);
Assert.NotNull(loaded);
// Incoming non-null wins — node-b replaces node-a on rank advance.
Assert.Equal("node-b", loaded!.SourceNode);
// Other monotonic fields advanced too — proves the UPDATE ran.
Assert.Equal("Attempted", loaded.Status);
}
}
[SkippableFact]
public async Task UpsertAsync_FillsSourceNode_WhenInsertWasNullAndLaterPacketCarriesValue()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
// SourceNode-stamping (Task 14): when the column was left NULL by an
// earlier unstamped packet, a later rank-advancing packet with a
// non-null SourceNode fills it — the COALESCE(@SourceNode, SourceNode)
// SQL operator returns @SourceNode when @SourceNode is non-null, so
// the incoming value wins over the existing NULL. This is the
// recovery path for an initially-unstamped lifecycle whose later
// packets carry the node identity.
//
// The intermediate verification and final read use FRESH contexts —
// FindAsync hits the change tracker first, so a cached entity from
// an earlier read in the same context can mask a raw-SQL UPDATE.
var id = TrackedOperationId.New();
await using (var context = CreateContext())
{
var repo = new SiteCallAuditRepository(context);
await repo.UpsertAsync(NewRow(id, status: "Submitted", sourceNode: null));
}
// Verify the INSERT left SourceNode NULL via a fresh context.
await using (var verifyContext = CreateContext())
{
var verifyRepo = new SiteCallAuditRepository(verifyContext);
var afterInsert = await verifyRepo.GetAsync(id);
Assert.NotNull(afterInsert);
Assert.Null(afterInsert!.SourceNode);
}
await using (var context = CreateContext())
{
var repo = new SiteCallAuditRepository(context);
await repo.UpsertAsync(NewRow(id, status: "Attempted", retryCount: 1, sourceNode: "node-a"));
}
await using (var readContext = CreateContext())
{
var readRepo = new SiteCallAuditRepository(readContext);
var loaded = await readRepo.GetAsync(id);
Assert.NotNull(loaded);
Assert.Equal("node-a", loaded!.SourceNode);
Assert.Equal("Attempted", loaded.Status);
}
}
}

View File

@@ -49,7 +49,8 @@ public class SiteCallAuditActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
string? lastError = null,
DateTime? createdAtUtc = null,
DateTime? updatedAtUtc = null,
bool terminal = false)
bool terminal = false,
string? sourceNode = null)
{
var created = createdAtUtc ?? DateTime.UtcNow;
var updated = updatedAtUtc ?? created;
@@ -59,6 +60,7 @@ public class SiteCallAuditActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
Channel = "ApiOutbound",
Target = "ERP.GetOrder",
SourceSite = sourceSite,
SourceNode = sourceNode,
Status = status,
RetryCount = retryCount,
LastError = lastError,
@@ -492,6 +494,67 @@ public class SiteCallAuditActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
Assert.Equal("corr-fault", response.CorrelationId);
}
// ── SourceNode-stamping (Task 14): end-to-end actor → repo persistence ──
[SkippableFact]
public async Task UpsertSiteCallCommand_PersistsSourceNode_EndToEnd()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
// SourceNode-stamping (Task 14): an UpsertSiteCallCommand carrying
// SourceNode "node-a" must land in the SiteCalls row's SourceNode
// column unchanged — verifies the actor's mapping path does not
// strip the column AND the repository INSERT writes it.
var siteId = NewSiteId();
var id = TrackedOperationId.New();
var row = NewRow(id, siteId, status: "Submitted", sourceNode: "node-a");
await using var context = CreateContext();
var repo = new SiteCallAuditRepository(context);
var actor = CreateActor(repo);
actor.Tell(new UpsertSiteCallCommand(row), TestActor);
var reply = ExpectMsg<UpsertSiteCallReply>(TimeSpan.FromSeconds(10));
Assert.True(reply.Accepted);
await using var readContext = CreateContext();
var stored = await readContext.Set<SiteCall>()
.Where(s => s.TrackedOperationId == id)
.ToListAsync();
Assert.Single(stored);
Assert.Equal("node-a", stored[0].SourceNode);
}
[SkippableFact]
public async Task UpsertSiteCallCommand_NullSourceNode_PersistsNull()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
// Mirror of the above for unstamped packets — a command with null
// SourceNode persists NULL on the row rather than falling back to a
// placeholder. The first audit packet from a legacy host (or a node
// without INodeIdentityProvider wired) must NOT inject a fabricated
// value central-side.
var siteId = NewSiteId();
var id = TrackedOperationId.New();
var row = NewRow(id, siteId, status: "Submitted", sourceNode: null);
await using var context = CreateContext();
var repo = new SiteCallAuditRepository(context);
var actor = CreateActor(repo);
actor.Tell(new UpsertSiteCallCommand(row), TestActor);
var reply = ExpectMsg<UpsertSiteCallReply>(TimeSpan.FromSeconds(10));
Assert.True(reply.Accepted);
await using var readContext = CreateContext();
var stored = await readContext.Set<SiteCall>()
.Where(s => s.TrackedOperationId == id)
.ToListAsync();
Assert.Single(stored);
Assert.Null(stored[0].SourceNode);
}
/// <summary>
/// Test double whose <see cref="ISiteCallAuditRepository.QueryAsync"/> always
/// throws — used to verify the query handler's failure projection produces a

View File

@@ -570,4 +570,69 @@ public class ExternalSystemCachedCallEmissionTests
var only = Assert.Single(forwarder.Telemetry);
Assert.Equal(AuditKind.CachedSubmit, only.Audit.Kind);
}
// ── SourceNode-stamping (Task 14) ──
[Fact]
public async Task CachedCall_StampsSourceNode_OnEverySiteCallOperationalRow()
{
// SourceNode-stamping (Task 14): when the helper is constructed with
// a non-null sourceNode, every SiteCallOperational it produces
// (CachedSubmit on enqueue + the immediate-completion Attempted/
// CachedResolve pair when WasBuffered=false) carries that node name.
var client = new Mock<IExternalSystemClient>();
client
.Setup(c => c.CachedCallAsync(
"ERP", "GetOrder",
It.IsAny<IReadOnlyDictionary<string, object?>?>(),
InstanceName,
It.IsAny<CancellationToken>(),
It.IsAny<TrackedOperationId?>(),
It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
// Immediate completion — helper produces all three rows itself.
.ReturnsAsync(new ExternalCallResult(true, "{\"ok\":true}", null, WasBuffered: false));
var forwarder = new CapturingForwarder();
var helper = new ScriptRuntimeContext.ExternalSystemHelper(
client.Object,
InstanceName,
NullLogger.Instance,
TestExecutionId,
auditWriter: null,
siteId: SiteId,
sourceScript: SourceScript,
cachedForwarder: forwarder,
parentExecutionId: null,
sourceNode: "node-a");
await helper.CachedCall("ERP", "GetOrder");
Assert.Equal(3, forwarder.Telemetry.Count);
Assert.All(forwarder.Telemetry, t => Assert.Equal("node-a", t.Operational.SourceNode));
}
[Fact]
public async Task CachedCall_NoSourceNodeWired_LeavesSourceNodeNull()
{
// Default CreateHelper does NOT pass sourceNode — the legacy / test
// host path. Every operational row carries null SourceNode, leaving
// central's SiteCalls.SourceNode NULL.
var client = new Mock<IExternalSystemClient>();
client
.Setup(c => c.CachedCallAsync(
"ERP", "GetOrder",
It.IsAny<IReadOnlyDictionary<string, object?>?>(),
InstanceName,
It.IsAny<CancellationToken>(),
It.IsAny<TrackedOperationId?>(),
It.IsAny<Guid?>(), It.IsAny<string?>(), It.IsAny<Guid?>()))
.ReturnsAsync(new ExternalCallResult(true, null, null, WasBuffered: true));
var forwarder = new CapturingForwarder();
var helper = CreateHelper(client.Object, forwarder);
await helper.CachedCall("ERP", "GetOrder");
var only = Assert.Single(forwarder.Telemetry);
Assert.Null(only.Operational.SourceNode);
}
}