refactor: rename ScadaLink → ZB.MOM.WW.ScadaBridge (code + projects + namespaces)

Solution + 23 src projects + 26 test projects renamed; folders, csproj,
namespaces, and ScadaLinkDbContext/ScadaBridgeDbContext class updated.
ActorSystem "scadalink" → "scadabridge", Akka seed-node URLs migrated.
SQL roles/logins, LDAP domains, CLI command name, and CLI config dir
(~/.scadalink → ~/.scadabridge) also renamed.

Build green; 5 Host.Tests fail awaiting SQL login rename in next commit.
Pre-existing StaleTagMonitor timing flakes unchanged.

Rename script committed at tools/rename-to-scadabridge.sh.
This commit is contained in:
Joseph Doherty
2026-05-28 09:37:45 -04:00
parent 6d87ee3c3b
commit 7b0b9c7365
1531 changed files with 11180 additions and 11054 deletions
@@ -0,0 +1,394 @@
using Microsoft.Extensions.Logging.Abstractions;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site.Telemetry;
/// <summary>
/// Bundle E Tasks E4/E5 bridge tests. The bridge ingests
/// <see cref="CachedCallAttemptContext"/> notifications from the S&amp;F
/// retry loop and routes them through <see cref="ICachedCallTelemetryForwarder"/>
/// as one or two <see cref="CachedCallTelemetry"/> packets:
/// <list type="bullet">
/// <item><description>Per-attempt: one <c>ApiCallCached</c>/<c>DbWriteCached</c> Attempted row.</description></item>
/// <item><description>Terminal (Delivered/PermanentFailure/ParkedMaxRetries): adds a CachedResolve row carrying the terminal Status.</description></item>
/// </list>
/// </summary>
public class CachedCallLifecycleBridgeTests
{
private readonly ICachedCallTelemetryForwarder _forwarder = Substitute.For<ICachedCallTelemetryForwarder>();
private readonly TrackedOperationId _id = TrackedOperationId.New();
private CachedCallLifecycleBridge CreateSut() => new(
_forwarder, NullLogger<CachedCallLifecycleBridge>.Instance);
private CachedCallAttemptContext Ctx(
CachedCallAttemptOutcome outcome,
string channel = "ApiOutbound",
int retryCount = 1,
string? lastError = null,
int? httpStatus = null,
Guid? executionId = null,
string? sourceScript = null,
Guid? parentExecutionId = null) =>
new(
TrackedOperationId: _id,
Channel: channel,
Target: "ERP.GetOrder",
SourceSite: "site-77",
Outcome: outcome,
RetryCount: retryCount,
LastError: lastError,
HttpStatus: httpStatus,
CreatedAtUtc: new DateTime(2026, 5, 20, 9, 0, 0, DateTimeKind.Utc),
OccurredAtUtc: new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc),
DurationMs: 42,
SourceInstanceId: "Plant.Pump42",
ExecutionId: executionId,
SourceScript: sourceScript,
ParentExecutionId: parentExecutionId);
[Fact]
public async Task TransientFailure_EmitsOneAttemptedRow_NoResolve()
{
var captured = new List<CachedCallTelemetry>();
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured.Add(t)), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
var sut = CreateSut();
await sut.OnAttemptCompletedAsync(Ctx(
CachedCallAttemptOutcome.TransientFailure,
retryCount: 2,
lastError: "HTTP 503",
httpStatus: 503));
var packet = Assert.Single(captured);
Assert.Equal(AuditKind.ApiCallCached, packet.Audit.Kind);
Assert.Equal(AuditStatus.Attempted, packet.Audit.Status);
Assert.Equal(503, packet.Audit.HttpStatus);
Assert.Equal("HTTP 503", packet.Audit.ErrorMessage);
Assert.Equal(_id.Value, packet.Audit.CorrelationId);
Assert.Equal("Attempted", packet.Operational.Status);
Assert.Equal(2, packet.Operational.RetryCount);
Assert.Null(packet.Operational.TerminalAtUtc);
}
[Fact]
public async Task Delivered_EmitsAttemptedRow_AndCachedResolveDelivered()
{
var captured = new List<CachedCallTelemetry>();
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured.Add(t)), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
var sut = CreateSut();
await sut.OnAttemptCompletedAsync(Ctx(CachedCallAttemptOutcome.Delivered));
Assert.Equal(2, captured.Count);
var attempted = captured[0];
Assert.Equal(AuditKind.ApiCallCached, attempted.Audit.Kind);
Assert.Equal(AuditStatus.Attempted, attempted.Audit.Status);
Assert.Equal("Attempted", attempted.Operational.Status);
Assert.Null(attempted.Operational.TerminalAtUtc);
var resolve = captured[1];
Assert.Equal(AuditKind.CachedResolve, resolve.Audit.Kind);
Assert.Equal(AuditStatus.Delivered, resolve.Audit.Status);
Assert.Equal("Delivered", resolve.Operational.Status);
Assert.NotNull(resolve.Operational.TerminalAtUtc);
Assert.Equal(_id.Value, resolve.Audit.CorrelationId);
}
[Fact]
public async Task PermanentFailure_EmitsAttempted_AndCachedResolveParked()
{
var captured = new List<CachedCallTelemetry>();
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured.Add(t)), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
var sut = CreateSut();
await sut.OnAttemptCompletedAsync(Ctx(
CachedCallAttemptOutcome.PermanentFailure,
lastError: "Permanent failure (handler returned false)"));
Assert.Equal(2, captured.Count);
Assert.Equal(AuditKind.ApiCallCached, captured[0].Audit.Kind);
Assert.Equal(AuditKind.CachedResolve, captured[1].Audit.Kind);
Assert.Equal(AuditStatus.Parked, captured[1].Audit.Status);
Assert.Equal("Parked", captured[1].Operational.Status);
}
[Fact]
public async Task ParkedMaxRetries_EmitsAttempted_AndCachedResolveParked()
{
var captured = new List<CachedCallTelemetry>();
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured.Add(t)), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
var sut = CreateSut();
await sut.OnAttemptCompletedAsync(Ctx(CachedCallAttemptOutcome.ParkedMaxRetries));
Assert.Equal(2, captured.Count);
Assert.Equal(AuditKind.CachedResolve, captured[1].Audit.Kind);
Assert.Equal(AuditStatus.Parked, captured[1].Audit.Status);
}
[Fact]
public async Task DbChannel_MapsToDbWriteCachedKind_AndDbOutboundChannel()
{
var captured = new List<CachedCallTelemetry>();
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured.Add(t)), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
var sut = CreateSut();
await sut.OnAttemptCompletedAsync(Ctx(
CachedCallAttemptOutcome.Delivered, channel: "DbOutbound"));
Assert.Equal(2, captured.Count);
Assert.Equal(AuditKind.DbWriteCached, captured[0].Audit.Kind);
Assert.Equal(AuditChannel.DbOutbound, captured[0].Audit.Channel);
Assert.Equal("DbOutbound", captured[0].Operational.Channel);
Assert.Equal(AuditKind.CachedResolve, captured[1].Audit.Kind);
Assert.Equal(AuditChannel.DbOutbound, captured[1].Audit.Channel);
}
[Fact]
public async Task BridgeDoesNotThrow_WhenForwarderThrows()
{
_forwarder
.ForwardAsync(Arg.Any<CachedCallTelemetry>(), Arg.Any<CancellationToken>())
.Returns(Task.FromException(new InvalidOperationException("forwarder down")));
var sut = CreateSut();
// Must not throw — best-effort emission.
await sut.OnAttemptCompletedAsync(Ctx(CachedCallAttemptOutcome.Delivered));
}
[Fact]
public async Task BridgePopulatesProvenance_FromAttemptContext()
{
CachedCallTelemetry? captured = null;
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured = t), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
var sut = CreateSut();
await sut.OnAttemptCompletedAsync(Ctx(
CachedCallAttemptOutcome.TransientFailure,
retryCount: 3,
lastError: "transient",
httpStatus: 500));
Assert.NotNull(captured);
Assert.Equal("site-77", captured!.Audit.SourceSiteId);
Assert.Equal("Plant.Pump42", captured.Audit.SourceInstanceId);
Assert.Equal("ERP.GetOrder", captured.Audit.Target);
Assert.Equal(42, captured.Audit.DurationMs);
Assert.Equal(_id.Value, captured.Audit.CorrelationId);
}
// ── Audit Log #23 (ExecutionId Task 4): ExecutionId / SourceScript ──
[Fact]
public async Task RetryLoopAttemptedRow_CarriesExecutionIdAndSourceScript_FromContext()
{
// Task 4: the ExecutionId + SourceScript threaded through the S&F
// buffer arrive on the CachedCallAttemptContext; the bridge must stamp
// both onto the per-attempt ApiCallCached row (previously SourceScript
// was hard-coded null with a "not threaded through S&F" comment).
var executionId = Guid.NewGuid();
var captured = new List<CachedCallTelemetry>();
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured.Add(t)), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
var sut = CreateSut();
await sut.OnAttemptCompletedAsync(Ctx(
CachedCallAttemptOutcome.TransientFailure,
executionId: executionId,
sourceScript: "Plant.Pump42/OnTick"));
var packet = Assert.Single(captured);
Assert.Equal(AuditKind.ApiCallCached, packet.Audit.Kind);
Assert.Equal(executionId, packet.Audit.ExecutionId);
Assert.Equal("Plant.Pump42/OnTick", packet.Audit.SourceScript);
}
[Fact]
public async Task RetryLoopCachedResolveRow_CarriesExecutionIdAndSourceScript_FromContext()
{
// The terminal CachedResolve row must also carry the threaded
// provenance so the whole retry-loop lifecycle is correlated.
var executionId = Guid.NewGuid();
var captured = new List<CachedCallTelemetry>();
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured.Add(t)), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
var sut = CreateSut();
await sut.OnAttemptCompletedAsync(Ctx(
CachedCallAttemptOutcome.Delivered,
channel: "DbOutbound",
executionId: executionId,
sourceScript: "Plant.Tank/OnAlarm"));
Assert.Equal(2, captured.Count);
var resolve = Assert.Single(captured, p => p.Audit.Kind == AuditKind.CachedResolve);
Assert.Equal(executionId, resolve.Audit.ExecutionId);
Assert.Equal("Plant.Tank/OnAlarm", resolve.Audit.SourceScript);
var attempted = Assert.Single(captured, p => p.Audit.Kind == AuditKind.DbWriteCached);
Assert.Equal(executionId, attempted.Audit.ExecutionId);
Assert.Equal("Plant.Tank/OnAlarm", attempted.Audit.SourceScript);
}
[Fact]
public async Task RetryLoopRow_NullExecutionIdAndSourceScript_RemainNull()
{
// Back-compat: a pre-Task-4 buffered row has no ExecutionId /
// SourceScript; the bridge must leave the audit row's fields null
// rather than throwing.
CachedCallTelemetry? captured = null;
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured = t), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
var sut = CreateSut();
await sut.OnAttemptCompletedAsync(Ctx(CachedCallAttemptOutcome.TransientFailure));
Assert.NotNull(captured);
Assert.Null(captured!.Audit.ExecutionId);
Assert.Null(captured.Audit.SourceScript);
}
// ── Audit Log #23 (ParentExecutionId Task 6): ParentExecutionId ──
[Fact]
public async Task RetryLoopAttemptedRow_CarriesParentExecutionId_FromContext()
{
// Task 6: the ParentExecutionId threaded through the S&F buffer (the
// inbound-API run that spawned the originating script) arrives on the
// CachedCallAttemptContext; the bridge must stamp it onto the
// per-attempt ApiCallCached row beside ExecutionId.
var parentExecutionId = Guid.NewGuid();
var captured = new List<CachedCallTelemetry>();
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured.Add(t)), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
var sut = CreateSut();
await sut.OnAttemptCompletedAsync(Ctx(
CachedCallAttemptOutcome.TransientFailure,
parentExecutionId: parentExecutionId));
var packet = Assert.Single(captured);
Assert.Equal(AuditKind.ApiCallCached, packet.Audit.Kind);
Assert.Equal(parentExecutionId, packet.Audit.ParentExecutionId);
}
[Fact]
public async Task RetryLoopCachedResolveRow_CarriesParentExecutionId_FromContext()
{
// The terminal CachedResolve row must also carry the threaded
// ParentExecutionId so the whole retry-loop lifecycle correlates back
// to the spawning inbound-API execution.
var parentExecutionId = Guid.NewGuid();
var captured = new List<CachedCallTelemetry>();
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured.Add(t)), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
var sut = CreateSut();
await sut.OnAttemptCompletedAsync(Ctx(
CachedCallAttemptOutcome.Delivered,
channel: "DbOutbound",
parentExecutionId: parentExecutionId));
Assert.Equal(2, captured.Count);
var resolve = Assert.Single(captured, p => p.Audit.Kind == AuditKind.CachedResolve);
Assert.Equal(parentExecutionId, resolve.Audit.ParentExecutionId);
var attempted = Assert.Single(captured, p => p.Audit.Kind == AuditKind.DbWriteCached);
Assert.Equal(parentExecutionId, attempted.Audit.ParentExecutionId);
}
[Fact]
public async Task RetryLoopRow_NullParentExecutionId_RemainsNull()
{
// Back-compat / non-routed run: the originating script was not spawned
// by an inbound-API request, so ParentExecutionId is null; the bridge
// must leave the audit row's ParentExecutionId null rather than throwing.
CachedCallTelemetry? captured = null;
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured = t), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
var sut = CreateSut();
await sut.OnAttemptCompletedAsync(Ctx(CachedCallAttemptOutcome.TransientFailure));
Assert.NotNull(captured);
Assert.Null(captured!.Audit.ParentExecutionId);
}
// ── SourceNode-stamping (Task 14) ──
[Fact]
public async Task RetryLoopRow_StampsSourceNode_FromNodeIdentityProvider()
{
// SourceNode-stamping (Task 14): when an INodeIdentityProvider is
// wired the bridge stamps the local node name (node-a/node-b) onto
// the SiteCallOperational.SourceNode column of every emitted packet.
var nodeIdentity = Substitute.For<INodeIdentityProvider>();
nodeIdentity.NodeName.Returns("node-a");
var captured = new List<CachedCallTelemetry>();
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured.Add(t)), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
var sut = new CachedCallLifecycleBridge(
_forwarder, NullLogger<CachedCallLifecycleBridge>.Instance, nodeIdentity);
await sut.OnAttemptCompletedAsync(Ctx(CachedCallAttemptOutcome.Delivered));
Assert.Equal(2, captured.Count);
Assert.All(captured, p => Assert.Equal("node-a", p.Operational.SourceNode));
}
[Fact]
public async Task RetryLoopRow_NoNodeIdentityProvider_LeavesSourceNodeNull()
{
// When no INodeIdentityProvider is wired (legacy hosts / tests) the
// bridge degrades to a null SourceNode rather than throwing. The
// emitted packet's SourceNode is null so the central row persists NULL.
var captured = new List<CachedCallTelemetry>();
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured.Add(t)), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
// Default CreateSut() does NOT pass a node-identity provider.
var sut = CreateSut();
await sut.OnAttemptCompletedAsync(Ctx(CachedCallAttemptOutcome.TransientFailure));
var packet = Assert.Single(captured);
Assert.Null(packet.Operational.SourceNode);
}
[Fact]
public async Task RetryLoopRow_NodeIdentityWithNullNodeName_LeavesSourceNodeNull()
{
// The provider exists but reports a null NodeName (unconfigured). The
// bridge must pass that null through to SourceNode rather than
// falling back to a placeholder.
var nodeIdentity = Substitute.For<INodeIdentityProvider>();
nodeIdentity.NodeName.Returns((string?)null);
var captured = new List<CachedCallTelemetry>();
_forwarder.ForwardAsync(Arg.Do<CachedCallTelemetry>(t => captured.Add(t)), Arg.Any<CancellationToken>())
.Returns(Task.CompletedTask);
var sut = new CachedCallLifecycleBridge(
_forwarder, NullLogger<CachedCallLifecycleBridge>.Instance, nodeIdentity);
await sut.OnAttemptCompletedAsync(Ctx(CachedCallAttemptOutcome.TransientFailure));
var packet = Assert.Single(captured);
Assert.Null(packet.Operational.SourceNode);
}
}
@@ -0,0 +1,307 @@
using Microsoft.Extensions.Logging.Abstractions;
using NSubstitute;
using NSubstitute.ExceptionExtensions;
using ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site.Telemetry;
/// <summary>
/// Bundle E E2 tests for <see cref="CachedCallTelemetryForwarder"/>. The
/// forwarder is the site-side dual emitter: every cached-call lifecycle event
/// writes one <see cref="AuditEvent"/> to <see cref="IAuditWriter"/> and one
/// operational tracking-row mutation to <see cref="IOperationTrackingStore"/>.
/// Audit-emission contract: best-effort — a thrown writer or tracking store
/// must be logged and swallowed; the forwarder must never propagate the
/// exception to the calling script.
/// </summary>
public class CachedCallTelemetryForwarderTests
{
private readonly IAuditWriter _writer = Substitute.For<IAuditWriter>();
private readonly IOperationTrackingStore _tracking = Substitute.For<IOperationTrackingStore>();
private readonly TrackedOperationId _id = TrackedOperationId.New();
private readonly DateTime _now = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc);
private CachedCallTelemetryForwarder CreateSut() => new(
_writer, _tracking, NullLogger<CachedCallTelemetryForwarder>.Instance);
private CachedCallTelemetry SubmitPacket() =>
new(
Audit: new AuditEvent
{
EventId = Guid.NewGuid(),
OccurredAtUtc = _now,
Channel = AuditChannel.ApiOutbound,
Kind = AuditKind.CachedSubmit,
CorrelationId = _id.Value,
SourceSiteId = "site-1",
SourceInstanceId = "inst-1",
SourceScript = "ScriptActor:doStuff",
Target = "ERP.GetOrder",
Status = AuditStatus.Submitted,
ForwardState = AuditForwardState.Pending,
},
Operational: new SiteCallOperational(
TrackedOperationId: _id,
Channel: "ApiOutbound",
Target: "ERP.GetOrder",
SourceSite: "site-1",
SourceNode: null,
Status: "Submitted",
RetryCount: 0,
LastError: null,
HttpStatus: null,
CreatedAtUtc: _now,
UpdatedAtUtc: _now,
TerminalAtUtc: null));
private CachedCallTelemetry AttemptedPacket(int retryCount = 1, string? lastError = "HTTP 500", int? httpStatus = 500) =>
new(
Audit: new AuditEvent
{
EventId = Guid.NewGuid(),
OccurredAtUtc = _now,
Channel = AuditChannel.ApiOutbound,
Kind = AuditKind.ApiCallCached,
CorrelationId = _id.Value,
SourceSiteId = "site-1",
Target = "ERP.GetOrder",
Status = AuditStatus.Attempted,
HttpStatus = httpStatus,
ErrorMessage = lastError,
ForwardState = AuditForwardState.Pending,
},
Operational: new SiteCallOperational(
TrackedOperationId: _id,
Channel: "ApiOutbound",
Target: "ERP.GetOrder",
SourceSite: "site-1",
SourceNode: null,
Status: "Attempted",
RetryCount: retryCount,
LastError: lastError,
HttpStatus: httpStatus,
CreatedAtUtc: _now,
UpdatedAtUtc: _now,
TerminalAtUtc: null));
private CachedCallTelemetry ResolvePacket(string status = "Delivered") =>
new(
Audit: new AuditEvent
{
EventId = Guid.NewGuid(),
OccurredAtUtc = _now,
Channel = AuditChannel.ApiOutbound,
Kind = AuditKind.CachedResolve,
CorrelationId = _id.Value,
SourceSiteId = "site-1",
Target = "ERP.GetOrder",
Status = Enum.Parse<AuditStatus>(status),
ForwardState = AuditForwardState.Pending,
},
Operational: new SiteCallOperational(
TrackedOperationId: _id,
Channel: "ApiOutbound",
Target: "ERP.GetOrder",
SourceSite: "site-1",
SourceNode: null,
Status: status,
RetryCount: 2,
LastError: null,
HttpStatus: null,
CreatedAtUtc: _now,
UpdatedAtUtc: _now,
TerminalAtUtc: _now));
[Fact]
public async Task ForwardAsync_Submit_WritesAuditEvent_AndRecordsEnqueue()
{
var sut = CreateSut();
var packet = SubmitPacket();
await sut.ForwardAsync(packet, CancellationToken.None);
// Audit row: one WriteAsync of the submit event.
await _writer.Received(1).WriteAsync(
Arg.Is<AuditEvent>(e =>
e.EventId == packet.Audit.EventId
&& e.Kind == AuditKind.CachedSubmit
&& e.Status == AuditStatus.Submitted),
Arg.Any<CancellationToken>());
// Tracking row: insert-if-not-exists with kind discriminator.
// Default CreateSut() does NOT supply an INodeIdentityProvider, so the
// forwarder passes null sourceNode to RecordEnqueueAsync (legacy / test
// host behaviour). The Task 14 stamping path is covered by the
// ForwardAsync_Submit_StampsSourceNode_FromNodeIdentityProvider test
// below.
await _tracking.Received(1).RecordEnqueueAsync(
_id,
"ApiOutbound",
"ERP.GetOrder",
"inst-1",
"ScriptActor:doStuff",
null,
Arg.Any<CancellationToken>());
await _tracking.DidNotReceiveWithAnyArgs().RecordAttemptAsync(
default, default!, default, default, default, default);
await _tracking.DidNotReceiveWithAnyArgs().RecordTerminalAsync(
default, default!, default, default, default);
}
[Fact]
public async Task ForwardAsync_Attempted_WritesAuditEvent_AndRecordsAttempt()
{
var sut = CreateSut();
var packet = AttemptedPacket(retryCount: 2, lastError: "HTTP 503", httpStatus: 503);
await sut.ForwardAsync(packet, CancellationToken.None);
await _writer.Received(1).WriteAsync(
Arg.Is<AuditEvent>(e =>
e.EventId == packet.Audit.EventId
&& e.Kind == AuditKind.ApiCallCached
&& e.Status == AuditStatus.Attempted),
Arg.Any<CancellationToken>());
await _tracking.Received(1).RecordAttemptAsync(
_id, "Attempted", 2, "HTTP 503", 503, Arg.Any<CancellationToken>());
await _tracking.DidNotReceiveWithAnyArgs().RecordEnqueueAsync(
default, default!, default, default, default, default, default);
await _tracking.DidNotReceiveWithAnyArgs().RecordTerminalAsync(
default, default!, default, default, default);
}
[Fact]
public async Task ForwardAsync_Resolve_WritesAuditEvent_AndRecordsTerminal()
{
var sut = CreateSut();
var packet = ResolvePacket("Delivered");
await sut.ForwardAsync(packet, CancellationToken.None);
await _writer.Received(1).WriteAsync(
Arg.Is<AuditEvent>(e =>
e.EventId == packet.Audit.EventId
&& e.Kind == AuditKind.CachedResolve
&& e.Status == AuditStatus.Delivered),
Arg.Any<CancellationToken>());
await _tracking.Received(1).RecordTerminalAsync(
_id, "Delivered", null, null, Arg.Any<CancellationToken>());
await _tracking.DidNotReceiveWithAnyArgs().RecordEnqueueAsync(
default, default!, default, default, default, default, default);
await _tracking.DidNotReceiveWithAnyArgs().RecordAttemptAsync(
default, default!, default, default, default, default);
}
[Fact]
public async Task ForwardAsync_WriterThrows_Logs_DoesNotPropagate()
{
_writer.WriteAsync(Arg.Any<AuditEvent>(), Arg.Any<CancellationToken>())
.Throws(new InvalidOperationException("primary down"));
var sut = CreateSut();
// Must not throw.
await sut.ForwardAsync(SubmitPacket(), CancellationToken.None);
// Tracking still attempted — emission of the two halves is independent
// so a writer outage cannot starve the operational row (and vice-versa).
await _tracking.Received(1).RecordEnqueueAsync(
Arg.Any<TrackedOperationId>(),
Arg.Any<string>(),
Arg.Any<string?>(),
Arg.Any<string?>(),
Arg.Any<string?>(),
Arg.Any<string?>(),
Arg.Any<CancellationToken>());
}
[Fact]
public async Task ForwardAsync_TrackingStoreThrows_Logs_DoesNotPropagate()
{
_tracking.RecordEnqueueAsync(
Arg.Any<TrackedOperationId>(),
Arg.Any<string>(),
Arg.Any<string?>(),
Arg.Any<string?>(),
Arg.Any<string?>(),
Arg.Any<string?>(),
Arg.Any<CancellationToken>())
.Throws(new InvalidOperationException("sqlite locked"));
var sut = CreateSut();
await sut.ForwardAsync(SubmitPacket(), CancellationToken.None);
// Writer still attempted — emission halves are independent.
await _writer.Received(1).WriteAsync(
Arg.Any<AuditEvent>(), Arg.Any<CancellationToken>());
}
[Fact]
public async Task ForwardAsync_NullPacket_Throws()
{
var sut = CreateSut();
await Assert.ThrowsAsync<ArgumentNullException>(
() => sut.ForwardAsync(null!, CancellationToken.None));
}
// ── SourceNode-stamping (Task 14) ──
[Fact]
public async Task ForwardAsync_Submit_StampsSourceNode_FromNodeIdentityProvider()
{
// SourceNode-stamping (Task 14): when an INodeIdentityProvider is
// wired the forwarder must stamp its NodeName onto the
// RecordEnqueueAsync sourceNode parameter so the tracking row
// captures the originating node (node-a/node-b).
var nodeIdentity = Substitute.For<INodeIdentityProvider>();
nodeIdentity.NodeName.Returns("node-a");
var sut = new CachedCallTelemetryForwarder(
_writer, _tracking, NullLogger<CachedCallTelemetryForwarder>.Instance, nodeIdentity);
await sut.ForwardAsync(SubmitPacket(), CancellationToken.None);
await _tracking.Received(1).RecordEnqueueAsync(
_id,
"ApiOutbound",
"ERP.GetOrder",
"inst-1",
"ScriptActor:doStuff",
"node-a",
Arg.Any<CancellationToken>());
}
[Fact]
public async Task ForwardAsync_Submit_NodeIdentityNullNodeName_PassesNullSourceNode()
{
// The provider exists but reports a null NodeName (unconfigured).
// The forwarder passes that null through to RecordEnqueueAsync rather
// than falling back to a placeholder string.
var nodeIdentity = Substitute.For<INodeIdentityProvider>();
nodeIdentity.NodeName.Returns((string?)null);
var sut = new CachedCallTelemetryForwarder(
_writer, _tracking, NullLogger<CachedCallTelemetryForwarder>.Instance, nodeIdentity);
await sut.ForwardAsync(SubmitPacket(), CancellationToken.None);
await _tracking.Received(1).RecordEnqueueAsync(
_id,
"ApiOutbound",
"ERP.GetOrder",
"inst-1",
"ScriptActor:doStuff",
null,
Arg.Any<CancellationToken>());
}
}
@@ -0,0 +1,201 @@
using Akka.Actor;
using Akka.TestKit.Xunit2;
using Google.Protobuf.WellKnownTypes;
using ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site.Telemetry;
/// <summary>
/// Tests for <see cref="ClusterClientSiteAuditClient"/> — the production
/// <see cref="ISiteStreamAuditClient"/> binding wired by the Host for site
/// roles. The client maps the proto-DTO batches produced by
/// <see cref="SiteAuditTelemetryActor"/> into the Akka
/// <see cref="IngestAuditEventsCommand"/> / <see cref="IngestCachedTelemetryCommand"/>
/// messages, Asks the site's <c>SiteCommunicationActor</c> (which forwards over
/// ClusterClient to central), and maps the reply back into an
/// <see cref="IngestAck"/>.
/// </summary>
/// <remarks>
/// A <see cref="TestProbe"/> stands in for the <c>SiteCommunicationActor</c>:
/// it lets the tests assert the exact command shape AND drive the reply (or
/// withhold one to exercise the Ask-timeout path).
/// </remarks>
public class ClusterClientSiteAuditClientTests : TestKit
{
/// <summary>Short Ask timeout so the timeout test completes quickly.</summary>
private static readonly TimeSpan AskTimeout = TimeSpan.FromMilliseconds(500);
private static AuditEvent NewEvent(Guid? id = null) => new()
{
EventId = id ?? Guid.NewGuid(),
OccurredAtUtc = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc),
Channel = AuditChannel.ApiOutbound,
Kind = AuditKind.ApiCall,
Status = AuditStatus.Delivered,
SourceSiteId = "site-1",
ForwardState = AuditForwardState.Pending,
};
private static AuditEventBatch BatchOf(IEnumerable<AuditEvent> events)
{
var batch = new AuditEventBatch();
foreach (var e in events)
{
batch.Events.Add(AuditEventDtoMapper.ToDto(e));
}
return batch;
}
private static SiteCallOperationalDto NewOperationalDto() => new()
{
TrackedOperationId = Guid.NewGuid().ToString(),
Channel = "ApiOutbound",
Target = "ext-system-1",
SourceSite = "site-1",
Status = "Submitted",
RetryCount = 0,
LastError = string.Empty,
CreatedAtUtc = Timestamp.FromDateTime(new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc)),
UpdatedAtUtc = Timestamp.FromDateTime(new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc)),
};
[Fact]
public async Task IngestAuditEventsAsync_FullAck_MapsAllAcceptedIdsOntoAck()
{
var probe = CreateTestProbe();
var sut = new ClusterClientSiteAuditClient(probe.Ref, AskTimeout);
var events = Enumerable.Range(0, 3).Select(_ => NewEvent()).ToList();
var batch = BatchOf(events);
var task = sut.IngestAuditEventsAsync(batch, CancellationToken.None);
// The probe receives exactly one IngestAuditEventsCommand carrying the
// batch's events; it replies with every EventId accepted.
var cmd = probe.ExpectMsg<IngestAuditEventsCommand>(TimeSpan.FromSeconds(3));
Assert.Equal(3, cmd.Events.Count);
Assert.Equal(
events.Select(e => e.EventId).ToHashSet(),
cmd.Events.Select(e => e.EventId).ToHashSet());
probe.Reply(new IngestAuditEventsReply(events.Select(e => e.EventId).ToList()));
var ack = await task;
Assert.Equal(
events.Select(e => e.EventId.ToString()).ToHashSet(),
ack.AcceptedEventIds.ToHashSet());
}
[Fact]
public async Task IngestAuditEventsAsync_PartialAck_OnlyAcceptedIdsAppearOnAck()
{
var probe = CreateTestProbe();
var sut = new ClusterClientSiteAuditClient(probe.Ref, AskTimeout);
var events = Enumerable.Range(0, 5).Select(_ => NewEvent()).ToList();
var accepted = events.Take(3).Select(e => e.EventId).ToList();
var task = sut.IngestAuditEventsAsync(BatchOf(events), CancellationToken.None);
probe.ExpectMsg<IngestAuditEventsCommand>(TimeSpan.FromSeconds(3));
probe.Reply(new IngestAuditEventsReply(accepted));
var ack = await task;
Assert.Equal(3, ack.AcceptedEventIds.Count);
Assert.Equal(
accepted.Select(id => id.ToString()).ToHashSet(),
ack.AcceptedEventIds.ToHashSet());
}
[Fact]
public async Task IngestAuditEventsAsync_AskTimeout_Throws_SoDrainLoopKeepsRowsPending()
{
var probe = CreateTestProbe();
var sut = new ClusterClientSiteAuditClient(probe.Ref, AskTimeout);
var batch = BatchOf(new[] { NewEvent() });
// The probe receives the command but never replies — the Ask times out.
// The contract: a timeout MUST surface as a thrown exception so the
// SiteAuditTelemetryActor drain loop leaves the rows Pending.
var task = sut.IngestAuditEventsAsync(batch, CancellationToken.None);
probe.ExpectMsg<IngestAuditEventsCommand>(TimeSpan.FromSeconds(3));
await Assert.ThrowsAnyAsync<Exception>(() => task);
}
[Fact]
public async Task IngestAuditEventsAsync_FaultedReply_Throws()
{
var probe = CreateTestProbe();
var sut = new ClusterClientSiteAuditClient(probe.Ref, AskTimeout);
var task = sut.IngestAuditEventsAsync(BatchOf(new[] { NewEvent() }), CancellationToken.None);
probe.ExpectMsg<IngestAuditEventsCommand>(TimeSpan.FromSeconds(3));
// A Status.Failure from central (Task 1: central does not swallow an
// ingest fault into an empty ack) must propagate as a thrown exception.
probe.Reply(new Status.Failure(new InvalidOperationException("central ingest faulted")));
await Assert.ThrowsAnyAsync<Exception>(() => task);
}
[Fact]
public async Task IngestCachedTelemetryAsync_RoutesCommand_AndMapsReply()
{
var probe = CreateTestProbe();
var sut = new ClusterClientSiteAuditClient(probe.Ref, AskTimeout);
var events = Enumerable.Range(0, 2).Select(_ => NewEvent()).ToList();
var batch = new CachedTelemetryBatch();
foreach (var e in events)
{
batch.Packets.Add(new CachedTelemetryPacket
{
AuditEvent = AuditEventDtoMapper.ToDto(e),
Operational = NewOperationalDto(),
});
}
var task = sut.IngestCachedTelemetryAsync(batch, CancellationToken.None);
// The probe receives an IngestCachedTelemetryCommand (NOT an
// IngestAuditEventsCommand) with one entry per packet.
var cmd = probe.ExpectMsg<IngestCachedTelemetryCommand>(TimeSpan.FromSeconds(3));
Assert.Equal(2, cmd.Entries.Count);
Assert.Equal(
events.Select(e => e.EventId).ToHashSet(),
cmd.Entries.Select(en => en.Audit.EventId).ToHashSet());
probe.Reply(new IngestCachedTelemetryReply(events.Select(e => e.EventId).ToList()));
var ack = await task;
Assert.Equal(
events.Select(e => e.EventId.ToString()).ToHashSet(),
ack.AcceptedEventIds.ToHashSet());
}
[Fact]
public async Task IngestCachedTelemetryAsync_AskTimeout_Throws()
{
var probe = CreateTestProbe();
var sut = new ClusterClientSiteAuditClient(probe.Ref, AskTimeout);
var batch = new CachedTelemetryBatch();
batch.Packets.Add(new CachedTelemetryPacket
{
AuditEvent = AuditEventDtoMapper.ToDto(NewEvent()),
Operational = NewOperationalDto(),
});
var task = sut.IngestCachedTelemetryAsync(batch, CancellationToken.None);
probe.ExpectMsg<IngestCachedTelemetryCommand>(TimeSpan.FromSeconds(3));
await Assert.ThrowsAnyAsync<Exception>(() => task);
}
}
@@ -0,0 +1,58 @@
using ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry;
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site.Telemetry;
/// <summary>
/// Bundle E E1 tests for <see cref="NoOpSiteStreamAuditClient"/>. The NoOp
/// client is the default <see cref="ISiteStreamAuditClient"/> binding until M6
/// delivers the gRPC-backed implementation; both <c>IngestAuditEventsAsync</c>
/// (M2) and <c>IngestCachedTelemetryAsync</c> (M3) must return an empty ack
/// (no rows flipped to Forwarded) without throwing or partially handling the
/// batch.
/// </summary>
public class NoOpSiteStreamAuditClientTests
{
[Fact]
public async Task IngestCachedTelemetryAsync_EmptyBatch_ReturnsEmptyAck()
{
var sut = new NoOpSiteStreamAuditClient();
var batch = new CachedTelemetryBatch();
var ack = await sut.IngestCachedTelemetryAsync(batch, CancellationToken.None);
Assert.NotNull(ack);
Assert.Empty(ack.AcceptedEventIds);
}
[Fact]
public async Task IngestCachedTelemetryAsync_PopulatedBatch_ReturnsEmptyAck()
{
var sut = new NoOpSiteStreamAuditClient();
var batch = new CachedTelemetryBatch();
batch.Packets.Add(new CachedTelemetryPacket
{
AuditEvent = new AuditEventDto
{
EventId = Guid.NewGuid().ToString(),
Channel = "ApiOutbound",
Kind = "CachedSubmit",
Status = "Submitted",
},
});
var ack = await sut.IngestCachedTelemetryAsync(batch, CancellationToken.None);
// No EventIds flipped — NoOp does not forward to anyone.
Assert.Empty(ack.AcceptedEventIds);
}
[Fact]
public async Task IngestCachedTelemetryAsync_NullBatch_Throws()
{
var sut = new NoOpSiteStreamAuditClient();
await Assert.ThrowsAsync<ArgumentNullException>(
() => sut.IngestCachedTelemetryAsync(null!, CancellationToken.None));
}
}
@@ -0,0 +1,456 @@
using Akka.Actor;
using Akka.TestKit.Xunit2;
using Google.Protobuf.WellKnownTypes;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NSubstitute;
using NSubstitute.ExceptionExtensions;
using ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site.Telemetry;
/// <summary>
/// Bundle D D1 tests for <see cref="SiteAuditTelemetryActor"/>. The actor drains
/// the site SQLite queue via <see cref="ISiteAuditQueue"/>, pushes batches via
/// <see cref="ISiteStreamAuditClient"/>, and flips ack'd rows to Forwarded.
/// Both collaborators are NSubstitute mocks so the tests never touch real
/// SQLite or gRPC.
/// </summary>
public class SiteAuditTelemetryActorTests : TestKit
{
private readonly ISiteAuditQueue _queue = Substitute.For<ISiteAuditQueue>();
private readonly ISiteStreamAuditClient _client = Substitute.For<ISiteStreamAuditClient>();
private readonly IOperationTrackingStore _trackingStore = Substitute.For<IOperationTrackingStore>();
/// <summary>
/// Fast options so tests don't stall waiting for the scheduler. 1s busy /
/// 2s idle still exercises the busy-vs-idle branching, but each test
/// completes in &lt; 5 s wall-clock.
/// </summary>
private static IOptions<SiteAuditTelemetryOptions> Opts(
int batchSize = 256,
int busySeconds = 1,
int idleSeconds = 2) =>
Options.Create(new SiteAuditTelemetryOptions
{
BatchSize = batchSize,
BusyIntervalSeconds = busySeconds,
IdleIntervalSeconds = idleSeconds,
});
private IActorRef CreateActor(IOptions<SiteAuditTelemetryOptions>? options = null) =>
Sys.ActorOf(Props.Create(() => new SiteAuditTelemetryActor(
_queue,
_client,
options ?? Opts(),
NullLogger<SiteAuditTelemetryActor>.Instance,
(IOperationTrackingStore?)null)));
/// <summary>
/// AuditLog-001: builds an actor with the optional
/// <see cref="IOperationTrackingStore"/> wired in so the cached-drain
/// scheduler is armed alongside the audit-only drain. Used by the new
/// cached-drain regression tests below.
/// </summary>
private IActorRef CreateActorWithCachedDrain(IOptions<SiteAuditTelemetryOptions>? options = null) =>
Sys.ActorOf(Props.Create(() => new SiteAuditTelemetryActor(
_queue,
_client,
options ?? Opts(),
NullLogger<SiteAuditTelemetryActor>.Instance,
(IOperationTrackingStore?)_trackingStore)));
private static AuditEvent NewEvent(Guid? id = null) => new()
{
EventId = id ?? Guid.NewGuid(),
OccurredAtUtc = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc),
Channel = AuditChannel.ApiOutbound,
Kind = AuditKind.ApiCall,
Status = AuditStatus.Delivered,
SourceSiteId = "site-1",
ForwardState = AuditForwardState.Pending,
};
private static IngestAck AckAll(IReadOnlyList<AuditEvent> events)
{
var ack = new IngestAck();
foreach (var e in events)
{
ack.AcceptedEventIds.Add(e.EventId.ToString());
}
return ack;
}
[Fact]
public async Task Drain_With_50PendingRows_Sends_OneBatch_Of_50_Then_FlipsToForwarded()
{
// Arrange — 50 pending rows on the first read, then empty on subsequent
// reads so the actor settles after one productive drain.
var pending = Enumerable.Range(0, 50).Select(_ => NewEvent()).ToList();
_queue.ReadPendingAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(
Task.FromResult<IReadOnlyList<AuditEvent>>(pending),
Task.FromResult<IReadOnlyList<AuditEvent>>(Array.Empty<AuditEvent>()));
AuditEventBatch? capturedBatch = null;
_client.IngestAuditEventsAsync(Arg.Any<AuditEventBatch>(), Arg.Any<CancellationToken>())
.Returns(call =>
{
capturedBatch = call.Arg<AuditEventBatch>();
return Task.FromResult(AckAll(pending));
});
// Act
CreateActor();
// Assert — give the scheduler time to fire the initial Drain tick.
await AwaitAssertAsync(async () =>
{
await _client.Received(1).IngestAuditEventsAsync(
Arg.Any<AuditEventBatch>(), Arg.Any<CancellationToken>());
await _queue.Received(1).MarkForwardedAsync(
Arg.Is<IReadOnlyList<Guid>>(g => g.Count == 50), Arg.Any<CancellationToken>());
}, TimeSpan.FromSeconds(5));
Assert.NotNull(capturedBatch);
Assert.Equal(50, capturedBatch!.Events.Count);
var expected = pending.Select(e => e.EventId).ToHashSet();
await _queue.Received(1).MarkForwardedAsync(
Arg.Is<IReadOnlyList<Guid>>(g => g.ToHashSet().SetEquals(expected)),
Arg.Any<CancellationToken>());
}
[Fact]
public async Task Drain_GrpcThrows_RowsStayPending_NextDrainRetries()
{
// Arrange — first read returns 3 rows; the gRPC client throws on the
// first push, then succeeds on the second. After the second push the
// queue returns empty so the actor settles.
var batch = Enumerable.Range(0, 3).Select(_ => NewEvent()).ToList();
_queue.ReadPendingAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(
Task.FromResult<IReadOnlyList<AuditEvent>>(batch),
Task.FromResult<IReadOnlyList<AuditEvent>>(batch),
Task.FromResult<IReadOnlyList<AuditEvent>>(Array.Empty<AuditEvent>()));
var calls = 0;
_client.IngestAuditEventsAsync(Arg.Any<AuditEventBatch>(), Arg.Any<CancellationToken>())
.Returns(_ =>
{
calls++;
if (calls == 1)
{
throw new InvalidOperationException("simulated gRPC failure");
}
return Task.FromResult(AckAll(batch));
});
// Act
CreateActor();
// Assert — eventually MarkForwardedAsync is called exactly once (after
// the retry succeeded). The first failure must NOT have called
// MarkForwardedAsync because the rows stay Pending.
await AwaitAssertAsync(async () =>
{
await _queue.Received(1).MarkForwardedAsync(
Arg.Any<IReadOnlyList<Guid>>(), Arg.Any<CancellationToken>());
}, TimeSpan.FromSeconds(10));
Assert.True(calls >= 2, $"Expected at least 2 client calls (1 failure + 1 retry); saw {calls}");
}
[Fact]
public async Task Drain_ZeroPending_SchedulesAtIdleInterval_NoClientCall()
{
// Arrange — queue always empty.
_queue.ReadPendingAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(Task.FromResult<IReadOnlyList<AuditEvent>>(Array.Empty<AuditEvent>()));
// Idle interval = 2 s. Pause 3 s after the first tick (1 s busy on
// PreStart) and assert the empty-queue branch did NOT push to the
// client.
CreateActor(Opts(busySeconds: 1, idleSeconds: 2));
// Allow the initial tick (~1 s) + a generous window for the idle re-tick.
await Task.Delay(TimeSpan.FromSeconds(3));
await _client.DidNotReceiveWithAnyArgs().IngestAuditEventsAsync(default!, default);
// ReadPendingAsync was called at least once (initial tick), and at
// most twice within the 3 s window (initial + one idle re-tick).
var readCalls = _queue.ReceivedCalls()
.Count(c => c.GetMethodInfo().Name == nameof(ISiteAuditQueue.ReadPendingAsync));
Assert.InRange(readCalls, 1, 2);
}
[Fact]
public async Task Drain_NonZeroPending_SchedulesAtBusyInterval()
{
// Arrange — every read returns 1 row. With busy=1s the actor should
// re-drain quickly, producing multiple client calls inside a short
// window.
var single = new List<AuditEvent> { NewEvent() };
_queue.ReadPendingAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(Task.FromResult<IReadOnlyList<AuditEvent>>(single));
_client.IngestAuditEventsAsync(Arg.Any<AuditEventBatch>(), Arg.Any<CancellationToken>())
.Returns(call => Task.FromResult(AckAll(single)));
CreateActor(Opts(busySeconds: 1, idleSeconds: 10));
// 3-second window with busy=1s should fit at least 2 drains.
await Task.Delay(TimeSpan.FromSeconds(3));
var pushCalls = _client.ReceivedCalls()
.Count(c => c.GetMethodInfo().Name == nameof(ISiteStreamAuditClient.IngestAuditEventsAsync));
Assert.True(pushCalls >= 2,
$"Expected ≥2 pushes within 3s when busy=1s; saw {pushCalls}");
}
[Fact]
public async Task Drain_AcceptedEventIdsSubset_OnlyMarksAccepted()
{
// Arrange — 5 rows pushed, but the central ack only lists 3.
var rows = Enumerable.Range(0, 5).Select(_ => NewEvent()).ToList();
var ackedIds = rows.Take(3).Select(r => r.EventId).ToList();
_queue.ReadPendingAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(
Task.FromResult<IReadOnlyList<AuditEvent>>(rows),
Task.FromResult<IReadOnlyList<AuditEvent>>(Array.Empty<AuditEvent>()));
var partialAck = new IngestAck();
foreach (var id in ackedIds)
{
partialAck.AcceptedEventIds.Add(id.ToString());
}
_client.IngestAuditEventsAsync(Arg.Any<AuditEventBatch>(), Arg.Any<CancellationToken>())
.Returns(Task.FromResult(partialAck));
// Act
CreateActor();
await AwaitAssertAsync(async () =>
{
await _queue.Received(1).MarkForwardedAsync(
Arg.Any<IReadOnlyList<Guid>>(), Arg.Any<CancellationToken>());
}, TimeSpan.FromSeconds(5));
// Assert — exactly the 3 ack'd ids made it to MarkForwardedAsync, not
// the other 2.
var ackedSet = ackedIds.ToHashSet();
await _queue.Received(1).MarkForwardedAsync(
Arg.Is<IReadOnlyList<Guid>>(g => g.Count == 3 && g.ToHashSet().SetEquals(ackedSet)),
Arg.Any<CancellationToken>());
}
// ────────────────────────────────────────────────────────────────────────
// AuditLog-001: combined-telemetry cached-drain regression tests. Verify
// that the production wiring of the previously-unreachable cached transport
// routes cached rows through ReadPendingCachedTelemetryAsync +
// IngestCachedTelemetryAsync (and NOT IngestAuditEventsAsync), and that
// orphaned audit rows (no tracking snapshot) are logged + skipped rather
// than crashing the drain.
// ────────────────────────────────────────────────────────────────────────
private static AuditEvent NewCachedEvent(
AuditKind kind = AuditKind.CachedSubmit,
Guid? eventId = null,
Guid? correlationId = null,
string sourceSiteId = "site-1") => new()
{
EventId = eventId ?? Guid.NewGuid(),
OccurredAtUtc = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc),
Channel = AuditChannel.ApiOutbound,
Kind = kind,
Status = AuditStatus.Submitted,
SourceSiteId = sourceSiteId,
Target = "ERP.GetOrder",
CorrelationId = correlationId ?? Guid.NewGuid(),
ForwardState = AuditForwardState.Pending,
};
private static TrackingStatusSnapshot NewSnapshot(
TrackedOperationId id,
string status = "Submitted",
int retryCount = 0) => new(
Id: id,
Kind: nameof(AuditKind.ApiCallCached),
TargetSummary: "ERP.GetOrder",
Status: status,
RetryCount: retryCount,
LastError: null,
HttpStatus: null,
CreatedAtUtc: new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc),
UpdatedAtUtc: new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc),
TerminalAtUtc: null,
SourceInstanceId: "instance-1",
SourceScript: "script-1",
SourceNode: "node-a");
[Fact]
public async Task CachedDrain_CachedRows_RouteThrough_IngestCachedTelemetry_NotIngestAuditEvents()
{
// Arrange — three cached audit rows on the cached queue, each with a
// matching tracking snapshot. The audit-only queue is empty (those
// rows are excluded by ReadPendingAsync after AuditLog-001).
var cachedRows = new[]
{
NewCachedEvent(AuditKind.CachedSubmit),
NewCachedEvent(AuditKind.ApiCallCached),
NewCachedEvent(AuditKind.CachedResolve),
};
_queue.ReadPendingAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(Task.FromResult<IReadOnlyList<AuditEvent>>(Array.Empty<AuditEvent>()));
_queue.ReadPendingCachedTelemetryAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(
Task.FromResult<IReadOnlyList<AuditEvent>>(cachedRows),
Task.FromResult<IReadOnlyList<AuditEvent>>(Array.Empty<AuditEvent>()));
foreach (var row in cachedRows)
{
var tid = new TrackedOperationId(row.CorrelationId!.Value);
_trackingStore.GetStatusAsync(tid, Arg.Any<CancellationToken>())
.Returns(Task.FromResult<TrackingStatusSnapshot?>(NewSnapshot(tid)));
}
CachedTelemetryBatch? capturedBatch = null;
_client.IngestCachedTelemetryAsync(Arg.Any<CachedTelemetryBatch>(), Arg.Any<CancellationToken>())
.Returns(call =>
{
capturedBatch = call.Arg<CachedTelemetryBatch>();
var ack = new IngestAck();
foreach (var packet in capturedBatch.Packets)
{
ack.AcceptedEventIds.Add(packet.AuditEvent.EventId);
}
return Task.FromResult(ack);
});
// Act
CreateActorWithCachedDrain();
// Assert — exactly one IngestCachedTelemetryAsync push containing all
// three packets, and zero IngestAuditEventsAsync pushes (the audit-only
// drain saw an empty queue).
await AwaitAssertAsync(async () =>
{
await _client.Received(1).IngestCachedTelemetryAsync(
Arg.Any<CachedTelemetryBatch>(), Arg.Any<CancellationToken>());
await _queue.Received(1).MarkForwardedAsync(
Arg.Is<IReadOnlyList<Guid>>(g => g.Count == 3), Arg.Any<CancellationToken>());
}, TimeSpan.FromSeconds(5));
Assert.NotNull(capturedBatch);
Assert.Equal(3, capturedBatch!.Packets.Count);
await _client.DidNotReceiveWithAnyArgs().IngestAuditEventsAsync(default!, default);
var emittedEventIds = capturedBatch.Packets
.Select(p => Guid.Parse(p.AuditEvent.EventId))
.ToHashSet();
var expectedIds = cachedRows.Select(r => r.EventId).ToHashSet();
Assert.Equal(expectedIds, emittedEventIds);
}
[Fact]
public async Task CachedDrain_OrphanRow_NoTrackingSnapshot_IsSkipped_DoesNotCrash()
{
// Arrange — two cached audit rows: one with a tracking snapshot, one
// orphaned (the tracking store returns null). The orphaned row must be
// skipped without aborting the batch — the valid row still flows.
var orphan = NewCachedEvent(AuditKind.CachedSubmit);
var valid = NewCachedEvent(AuditKind.CachedResolve);
_queue.ReadPendingAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(Task.FromResult<IReadOnlyList<AuditEvent>>(Array.Empty<AuditEvent>()));
_queue.ReadPendingCachedTelemetryAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(
Task.FromResult<IReadOnlyList<AuditEvent>>(new[] { orphan, valid }),
Task.FromResult<IReadOnlyList<AuditEvent>>(Array.Empty<AuditEvent>()));
// orphan: tracking returns null
_trackingStore.GetStatusAsync(
new TrackedOperationId(orphan.CorrelationId!.Value),
Arg.Any<CancellationToken>())
.Returns(Task.FromResult<TrackingStatusSnapshot?>(null));
// valid: tracking returns a snapshot
var validTid = new TrackedOperationId(valid.CorrelationId!.Value);
_trackingStore.GetStatusAsync(validTid, Arg.Any<CancellationToken>())
.Returns(Task.FromResult<TrackingStatusSnapshot?>(NewSnapshot(validTid, "Delivered")));
CachedTelemetryBatch? capturedBatch = null;
_client.IngestCachedTelemetryAsync(Arg.Any<CachedTelemetryBatch>(), Arg.Any<CancellationToken>())
.Returns(call =>
{
capturedBatch = call.Arg<CachedTelemetryBatch>();
var ack = new IngestAck();
foreach (var packet in capturedBatch.Packets)
{
ack.AcceptedEventIds.Add(packet.AuditEvent.EventId);
}
return Task.FromResult(ack);
});
// Act
CreateActorWithCachedDrain();
// Assert — exactly one push containing ONLY the valid row; the orphan
// is skipped and stays Pending (not in MarkForwardedAsync's id list).
await AwaitAssertAsync(async () =>
{
await _client.Received(1).IngestCachedTelemetryAsync(
Arg.Any<CachedTelemetryBatch>(), Arg.Any<CancellationToken>());
}, TimeSpan.FromSeconds(5));
Assert.NotNull(capturedBatch);
Assert.Single(capturedBatch!.Packets);
Assert.Equal(valid.EventId.ToString(), capturedBatch.Packets[0].AuditEvent.EventId);
await _queue.Received(1).MarkForwardedAsync(
Arg.Is<IReadOnlyList<Guid>>(g => g.Count == 1 && g[0] == valid.EventId),
Arg.Any<CancellationToken>());
}
[Fact]
public async Task AuditOnlyDrain_StillFlows_When_CachedDrain_IsDisabled()
{
// Arrange — ordinary (non-cached) audit rows on the audit-only queue;
// the actor is constructed WITHOUT a tracking store so the cached
// scheduler is never armed. Regression guard against the audit-only
// drain regressing during the AuditLog-001 refactor.
var rows = Enumerable.Range(0, 3).Select(_ => NewEvent()).ToList();
_queue.ReadPendingAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(
Task.FromResult<IReadOnlyList<AuditEvent>>(rows),
Task.FromResult<IReadOnlyList<AuditEvent>>(Array.Empty<AuditEvent>()));
_client.IngestAuditEventsAsync(Arg.Any<AuditEventBatch>(), Arg.Any<CancellationToken>())
.Returns(_ => Task.FromResult(AckAll(rows)));
// Act — note: CreateActor (no tracking store), not CreateActorWithCachedDrain.
CreateActor();
// Assert — audit-only drain flows normally; the cached client is
// never called and ReadPendingCachedTelemetryAsync is never queried.
await AwaitAssertAsync(async () =>
{
await _client.Received(1).IngestAuditEventsAsync(
Arg.Any<AuditEventBatch>(), Arg.Any<CancellationToken>());
await _queue.Received(1).MarkForwardedAsync(
Arg.Is<IReadOnlyList<Guid>>(g => g.Count == 3), Arg.Any<CancellationToken>());
}, TimeSpan.FromSeconds(5));
await _client.DidNotReceiveWithAnyArgs().IngestCachedTelemetryAsync(default!, default);
await _queue.DidNotReceiveWithAnyArgs().ReadPendingCachedTelemetryAsync(default, default);
}
}