using Microsoft.Extensions.Logging.Abstractions; using NSubstitute; using NSubstitute.ExceptionExtensions; using ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry; using ZB.MOM.WW.Audit; using IAuditWriter = ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services.IAuditWriter; using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration; using ZB.MOM.WW.ScadaBridge.Commons.Types; using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site.Telemetry; /// /// Bundle E E2 tests for . The /// forwarder is the site-side dual emitter: every cached-call lifecycle event /// writes one to and one /// operational tracking-row mutation to . /// Audit-emission contract: best-effort — a thrown writer or tracking store /// must be logged and swallowed; the forwarder must never propagate the /// exception to the calling script. /// public class CachedCallTelemetryForwarderTests { private readonly IAuditWriter _writer = Substitute.For(); private readonly IOperationTrackingStore _tracking = Substitute.For(); private readonly TrackedOperationId _id = TrackedOperationId.New(); private readonly DateTime _now = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc); private CachedCallTelemetryForwarder CreateSut() => new( _writer, _tracking, NullLogger.Instance); private CachedCallTelemetry SubmitPacket() => new( Audit: ScadaBridgeAuditEventFactory.Create( eventId: Guid.NewGuid(), occurredAtUtc: _now, channel: AuditChannel.ApiOutbound, kind: AuditKind.CachedSubmit, correlationId: _id.Value, sourceSiteId: "site-1", sourceInstanceId: "inst-1", sourceScript: "ScriptActor:doStuff", target: "ERP.GetOrder", status: AuditStatus.Submitted), Operational: new SiteCallOperational( TrackedOperationId: _id, Channel: "ApiOutbound", Target: "ERP.GetOrder", SourceSite: "site-1", SourceNode: null, Status: "Submitted", RetryCount: 0, LastError: null, HttpStatus: null, CreatedAtUtc: _now, UpdatedAtUtc: _now, TerminalAtUtc: null)); private CachedCallTelemetry AttemptedPacket(int retryCount = 1, string? lastError = "HTTP 500", int? httpStatus = 500) => new( Audit: ScadaBridgeAuditEventFactory.Create( eventId: Guid.NewGuid(), occurredAtUtc: _now, channel: AuditChannel.ApiOutbound, kind: AuditKind.ApiCallCached, correlationId: _id.Value, sourceSiteId: "site-1", target: "ERP.GetOrder", status: AuditStatus.Attempted, httpStatus: httpStatus, errorMessage: lastError), Operational: new SiteCallOperational( TrackedOperationId: _id, Channel: "ApiOutbound", Target: "ERP.GetOrder", SourceSite: "site-1", SourceNode: null, Status: "Attempted", RetryCount: retryCount, LastError: lastError, HttpStatus: httpStatus, CreatedAtUtc: _now, UpdatedAtUtc: _now, TerminalAtUtc: null)); private CachedCallTelemetry ResolvePacket(string status = "Delivered") => new( Audit: ScadaBridgeAuditEventFactory.Create( eventId: Guid.NewGuid(), occurredAtUtc: _now, channel: AuditChannel.ApiOutbound, kind: AuditKind.CachedResolve, correlationId: _id.Value, sourceSiteId: "site-1", target: "ERP.GetOrder", status: Enum.Parse(status)), Operational: new SiteCallOperational( TrackedOperationId: _id, Channel: "ApiOutbound", Target: "ERP.GetOrder", SourceSite: "site-1", SourceNode: null, Status: status, RetryCount: 2, LastError: null, HttpStatus: null, CreatedAtUtc: _now, UpdatedAtUtc: _now, TerminalAtUtc: _now)); /// Verifies that forwarding a submit packet writes an audit event and records the enqueue. /// A task that represents the asynchronous operation. [Fact] public async Task ForwardAsync_Submit_WritesAuditEvent_AndRecordsEnqueue() { var sut = CreateSut(); var packet = SubmitPacket(); await sut.ForwardAsync(packet, CancellationToken.None); // Audit row: one WriteAsync of the submit event. await _writer.Received(1).WriteAsync( Arg.Is(e => e.EventId == packet.Audit.EventId && e.AsRow().Kind == AuditKind.CachedSubmit && e.AsRow().Status == AuditStatus.Submitted), Arg.Any()); // Tracking row: insert-if-not-exists with kind discriminator. // Default CreateSut() does NOT supply an INodeIdentityProvider, so the // forwarder passes null sourceNode to RecordEnqueueAsync (legacy / test // host behaviour). The stamping path is covered by the // ForwardAsync_Submit_StampsSourceNode_FromNodeIdentityProvider test // below. await _tracking.Received(1).RecordEnqueueAsync( _id, "ApiOutbound", "ERP.GetOrder", "inst-1", "ScriptActor:doStuff", null, Arg.Any()); await _tracking.DidNotReceiveWithAnyArgs().RecordAttemptAsync( default, default!, default, default, default, default); await _tracking.DidNotReceiveWithAnyArgs().RecordTerminalAsync( default, default!, default, default, default); } /// Verifies that forwarding an attempted packet writes an audit event and records the attempt. /// A task that represents the asynchronous operation. [Fact] public async Task ForwardAsync_Attempted_WritesAuditEvent_AndRecordsAttempt() { var sut = CreateSut(); var packet = AttemptedPacket(retryCount: 2, lastError: "HTTP 503", httpStatus: 503); await sut.ForwardAsync(packet, CancellationToken.None); await _writer.Received(1).WriteAsync( Arg.Is(e => e.EventId == packet.Audit.EventId && e.AsRow().Kind == AuditKind.ApiCallCached && e.AsRow().Status == AuditStatus.Attempted), Arg.Any()); await _tracking.Received(1).RecordAttemptAsync( _id, "Attempted", 2, "HTTP 503", 503, Arg.Any()); await _tracking.DidNotReceiveWithAnyArgs().RecordEnqueueAsync( default, default!, default, default, default, default, default); await _tracking.DidNotReceiveWithAnyArgs().RecordTerminalAsync( default, default!, default, default, default); } /// Verifies that forwarding a resolve packet writes an audit event and records the terminal state. /// A task that represents the asynchronous operation. [Fact] public async Task ForwardAsync_Resolve_WritesAuditEvent_AndRecordsTerminal() { var sut = CreateSut(); var packet = ResolvePacket("Delivered"); await sut.ForwardAsync(packet, CancellationToken.None); await _writer.Received(1).WriteAsync( Arg.Is(e => e.EventId == packet.Audit.EventId && e.AsRow().Kind == AuditKind.CachedResolve && e.AsRow().Status == AuditStatus.Delivered), Arg.Any()); await _tracking.Received(1).RecordTerminalAsync( _id, "Delivered", null, null, Arg.Any()); await _tracking.DidNotReceiveWithAnyArgs().RecordEnqueueAsync( default, default!, default, default, default, default, default); await _tracking.DidNotReceiveWithAnyArgs().RecordAttemptAsync( default, default!, default, default, default, default); } // ── 2026-09-06: the audit row is enqueued, never awaited to commit ───── // Design: docs/plans/2026-09-06-tracking-store-async-writes-design.md /// Verifies that forwarding returns while the audit commit is still pending. /// A task that represents the asynchronous operation. [Fact] public async Task ForwardAsync_ReturnsWhileTheAuditCommitIsStillPending() { // SqliteAuditWriter.WriteAsync returns the batch completion, which lands only // after FlushIntervalMs + fsync. A never-completing task models "not flushed // yet": ForwardAsync must still return and still record the tracking half. var neverFlushed = new TaskCompletionSource(); _writer.WriteAsync(Arg.Any(), Arg.Any()) .Returns(neverFlushed.Task); var sut = CreateSut(); var forward = sut.ForwardAsync(ResolvePacket(), CancellationToken.None); var finished = await Task.WhenAny(forward, Task.Delay(TimeSpan.FromSeconds(2))); Assert.Same(forward, finished); await _writer.Received(1).WriteAsync(Arg.Any(), Arg.Any()); await _tracking.Received(1).RecordTerminalAsync( _id, "Delivered", Arg.Any(), Arg.Any(), Arg.Any()); } /// Verifies that a later audit commit fault is swallowed rather than surfaced to the caller. /// A task that represents the asynchronous operation. [Fact] public async Task ForwardAsync_AuditCommitFaultsLater_IsSwallowed() { var lateFault = new TaskCompletionSource(); _writer.WriteAsync(Arg.Any(), Arg.Any()) .Returns(lateFault.Task); var sut = CreateSut(); await sut.ForwardAsync(SubmitPacket(), CancellationToken.None); // returns before the fault lateFault.SetException(new InvalidOperationException("disk full")); await Task.Yield(); // Nothing propagates anywhere the script could see; the tracking half ran. await _tracking.Received(1).RecordEnqueueAsync( Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()); } /// Verifies that a throwing audit writer is logged and the exception does not propagate. /// A task that represents the asynchronous operation. [Fact] public async Task ForwardAsync_WriterThrows_Logs_DoesNotPropagate() { _writer.WriteAsync(Arg.Any(), Arg.Any()) .Throws(new InvalidOperationException("primary down")); var sut = CreateSut(); // Must not throw. await sut.ForwardAsync(SubmitPacket(), CancellationToken.None); // Tracking still attempted — emission of the two halves is independent // so a writer outage cannot starve the operational row (and vice-versa). await _tracking.Received(1).RecordEnqueueAsync( Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()); } /// Verifies that a throwing tracking store is logged and the exception does not propagate. /// A task that represents the asynchronous operation. [Fact] public async Task ForwardAsync_TrackingStoreThrows_Logs_DoesNotPropagate() { _tracking.RecordEnqueueAsync( Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Throws(new InvalidOperationException("sqlite locked")); var sut = CreateSut(); await sut.ForwardAsync(SubmitPacket(), CancellationToken.None); // Writer still attempted — emission halves are independent. await _writer.Received(1).WriteAsync( Arg.Any(), Arg.Any()); } /// Verifies that forwarding a null packet throws. /// A task that represents the asynchronous operation. [Fact] public async Task ForwardAsync_NullPacket_Throws() { var sut = CreateSut(); await Assert.ThrowsAsync( () => sut.ForwardAsync(null!, CancellationToken.None)); } // ── SourceNode stamping ── /// Verifies that a submit packet is stamped with the source node from the node identity provider. /// A task that represents the asynchronous operation. [Fact] public async Task ForwardAsync_Submit_StampsSourceNode_FromNodeIdentityProvider() { // SourceNode stamping: when an INodeIdentityProvider is // wired the forwarder must stamp its NodeName onto the // RecordEnqueueAsync sourceNode parameter so the tracking row // captures the originating node (node-a/node-b). var nodeIdentity = Substitute.For(); nodeIdentity.NodeName.Returns("node-a"); var sut = new CachedCallTelemetryForwarder( _writer, _tracking, NullLogger.Instance, nodeIdentity); await sut.ForwardAsync(SubmitPacket(), CancellationToken.None); await _tracking.Received(1).RecordEnqueueAsync( _id, "ApiOutbound", "ERP.GetOrder", "inst-1", "ScriptActor:doStuff", "node-a", Arg.Any()); } /// Verifies that a node identity with a null node name results in a null source node. /// A task that represents the asynchronous operation. [Fact] public async Task ForwardAsync_Submit_NodeIdentityNullNodeName_PassesNullSourceNode() { // The provider exists but reports a null NodeName (unconfigured). // The forwarder passes that null through to RecordEnqueueAsync rather // than falling back to a placeholder string. var nodeIdentity = Substitute.For(); nodeIdentity.NodeName.Returns((string?)null); var sut = new CachedCallTelemetryForwarder( _writer, _tracking, NullLogger.Instance, nodeIdentity); await sut.ForwardAsync(SubmitPacket(), CancellationToken.None); await _tracking.Received(1).RecordEnqueueAsync( _id, "ApiOutbound", "ERP.GetOrder", "inst-1", "ScriptActor:doStuff", null, Arg.Any()); } }