Joseph Doherty
513fa1aa16
Comment/XML-doc changes only: - add summaries to ~10k test methods, fixtures, fakes and helpers (+ <param> on theories, <returns> on async Task methods, <inheritdoc/> on interface implementations the checker can resolve) - strip ~2.6k project-tracking references (WP*, P*-*, row-NN, Task N, #NN, Component-0NN) from comments, keeping the explanatory text; reword false positives in comments only - real docs on fakes implementing BCL/library interfaces (IDisposable, ILogger, IOptionsMonitor, IAsyncLifetime) - reword the last 5 src/ false positives (CWE-1236, failures-1, year-0001, M365) Only non-comment edits: four one-line nested classes split across lines so members can carry docs. Build 0 errors / 0 warnings.
362 lines
16 KiB
C#
362 lines
16 KiB
C#
using Microsoft.Extensions.Logging.Abstractions;
|
|
using NSubstitute;
|
|
using NSubstitute.ExceptionExtensions;
|
|
using ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry;
|
|
using ZB.MOM.WW.Audit;
|
|
using IAuditWriter = ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services.IAuditWriter;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site.Telemetry;
|
|
|
|
/// <summary>
|
|
/// Bundle E E2 tests for <see cref="CachedCallTelemetryForwarder"/>. The
|
|
/// forwarder is the site-side dual emitter: every cached-call lifecycle event
|
|
/// writes one <see cref="AuditEvent"/> to <see cref="IAuditWriter"/> and one
|
|
/// operational tracking-row mutation to <see cref="IOperationTrackingStore"/>.
|
|
/// Audit-emission contract: best-effort — a thrown writer or tracking store
|
|
/// must be logged and swallowed; the forwarder must never propagate the
|
|
/// exception to the calling script.
|
|
/// </summary>
|
|
public class CachedCallTelemetryForwarderTests
|
|
{
|
|
private readonly IAuditWriter _writer = Substitute.For<IAuditWriter>();
|
|
private readonly IOperationTrackingStore _tracking = Substitute.For<IOperationTrackingStore>();
|
|
private readonly TrackedOperationId _id = TrackedOperationId.New();
|
|
private readonly DateTime _now = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc);
|
|
|
|
private CachedCallTelemetryForwarder CreateSut() => new(
|
|
_writer, _tracking, NullLogger<CachedCallTelemetryForwarder>.Instance);
|
|
|
|
private CachedCallTelemetry SubmitPacket() =>
|
|
new(
|
|
Audit: ScadaBridgeAuditEventFactory.Create(
|
|
eventId: Guid.NewGuid(),
|
|
occurredAtUtc: _now,
|
|
channel: AuditChannel.ApiOutbound,
|
|
kind: AuditKind.CachedSubmit,
|
|
correlationId: _id.Value,
|
|
sourceSiteId: "site-1",
|
|
sourceInstanceId: "inst-1",
|
|
sourceScript: "ScriptActor:doStuff",
|
|
target: "ERP.GetOrder",
|
|
status: AuditStatus.Submitted),
|
|
Operational: new SiteCallOperational(
|
|
TrackedOperationId: _id,
|
|
Channel: "ApiOutbound",
|
|
Target: "ERP.GetOrder",
|
|
SourceSite: "site-1",
|
|
SourceNode: null,
|
|
Status: "Submitted",
|
|
RetryCount: 0,
|
|
LastError: null,
|
|
HttpStatus: null,
|
|
CreatedAtUtc: _now,
|
|
UpdatedAtUtc: _now,
|
|
TerminalAtUtc: null));
|
|
|
|
private CachedCallTelemetry AttemptedPacket(int retryCount = 1, string? lastError = "HTTP 500", int? httpStatus = 500) =>
|
|
new(
|
|
Audit: ScadaBridgeAuditEventFactory.Create(
|
|
eventId: Guid.NewGuid(),
|
|
occurredAtUtc: _now,
|
|
channel: AuditChannel.ApiOutbound,
|
|
kind: AuditKind.ApiCallCached,
|
|
correlationId: _id.Value,
|
|
sourceSiteId: "site-1",
|
|
target: "ERP.GetOrder",
|
|
status: AuditStatus.Attempted,
|
|
httpStatus: httpStatus,
|
|
errorMessage: lastError),
|
|
Operational: new SiteCallOperational(
|
|
TrackedOperationId: _id,
|
|
Channel: "ApiOutbound",
|
|
Target: "ERP.GetOrder",
|
|
SourceSite: "site-1",
|
|
SourceNode: null,
|
|
Status: "Attempted",
|
|
RetryCount: retryCount,
|
|
LastError: lastError,
|
|
HttpStatus: httpStatus,
|
|
CreatedAtUtc: _now,
|
|
UpdatedAtUtc: _now,
|
|
TerminalAtUtc: null));
|
|
|
|
private CachedCallTelemetry ResolvePacket(string status = "Delivered") =>
|
|
new(
|
|
Audit: ScadaBridgeAuditEventFactory.Create(
|
|
eventId: Guid.NewGuid(),
|
|
occurredAtUtc: _now,
|
|
channel: AuditChannel.ApiOutbound,
|
|
kind: AuditKind.CachedResolve,
|
|
correlationId: _id.Value,
|
|
sourceSiteId: "site-1",
|
|
target: "ERP.GetOrder",
|
|
status: Enum.Parse<AuditStatus>(status)),
|
|
Operational: new SiteCallOperational(
|
|
TrackedOperationId: _id,
|
|
Channel: "ApiOutbound",
|
|
Target: "ERP.GetOrder",
|
|
SourceSite: "site-1",
|
|
SourceNode: null,
|
|
Status: status,
|
|
RetryCount: 2,
|
|
LastError: null,
|
|
HttpStatus: null,
|
|
CreatedAtUtc: _now,
|
|
UpdatedAtUtc: _now,
|
|
TerminalAtUtc: _now));
|
|
|
|
/// <summary>Verifies that forwarding a submit packet writes an audit event and records the enqueue.</summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task ForwardAsync_Submit_WritesAuditEvent_AndRecordsEnqueue()
|
|
{
|
|
var sut = CreateSut();
|
|
var packet = SubmitPacket();
|
|
|
|
await sut.ForwardAsync(packet, CancellationToken.None);
|
|
|
|
// Audit row: one WriteAsync of the submit event.
|
|
await _writer.Received(1).WriteAsync(
|
|
Arg.Is<AuditEvent>(e =>
|
|
e.EventId == packet.Audit.EventId
|
|
&& e.AsRow().Kind == AuditKind.CachedSubmit
|
|
&& e.AsRow().Status == AuditStatus.Submitted),
|
|
Arg.Any<CancellationToken>());
|
|
|
|
// Tracking row: insert-if-not-exists with kind discriminator.
|
|
// Default CreateSut() does NOT supply an INodeIdentityProvider, so the
|
|
// forwarder passes null sourceNode to RecordEnqueueAsync (legacy / test
|
|
// host behaviour). The stamping path is covered by the
|
|
// ForwardAsync_Submit_StampsSourceNode_FromNodeIdentityProvider test
|
|
// below.
|
|
await _tracking.Received(1).RecordEnqueueAsync(
|
|
_id,
|
|
"ApiOutbound",
|
|
"ERP.GetOrder",
|
|
"inst-1",
|
|
"ScriptActor:doStuff",
|
|
null,
|
|
Arg.Any<CancellationToken>());
|
|
await _tracking.DidNotReceiveWithAnyArgs().RecordAttemptAsync(
|
|
default, default!, default, default, default, default);
|
|
await _tracking.DidNotReceiveWithAnyArgs().RecordTerminalAsync(
|
|
default, default!, default, default, default);
|
|
}
|
|
|
|
/// <summary>Verifies that forwarding an attempted packet writes an audit event and records the attempt.</summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task ForwardAsync_Attempted_WritesAuditEvent_AndRecordsAttempt()
|
|
{
|
|
var sut = CreateSut();
|
|
var packet = AttemptedPacket(retryCount: 2, lastError: "HTTP 503", httpStatus: 503);
|
|
|
|
await sut.ForwardAsync(packet, CancellationToken.None);
|
|
|
|
await _writer.Received(1).WriteAsync(
|
|
Arg.Is<AuditEvent>(e =>
|
|
e.EventId == packet.Audit.EventId
|
|
&& e.AsRow().Kind == AuditKind.ApiCallCached
|
|
&& e.AsRow().Status == AuditStatus.Attempted),
|
|
Arg.Any<CancellationToken>());
|
|
|
|
await _tracking.Received(1).RecordAttemptAsync(
|
|
_id, "Attempted", 2, "HTTP 503", 503, Arg.Any<CancellationToken>());
|
|
await _tracking.DidNotReceiveWithAnyArgs().RecordEnqueueAsync(
|
|
default, default!, default, default, default, default, default);
|
|
await _tracking.DidNotReceiveWithAnyArgs().RecordTerminalAsync(
|
|
default, default!, default, default, default);
|
|
}
|
|
|
|
/// <summary>Verifies that forwarding a resolve packet writes an audit event and records the terminal state.</summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task ForwardAsync_Resolve_WritesAuditEvent_AndRecordsTerminal()
|
|
{
|
|
var sut = CreateSut();
|
|
var packet = ResolvePacket("Delivered");
|
|
|
|
await sut.ForwardAsync(packet, CancellationToken.None);
|
|
|
|
await _writer.Received(1).WriteAsync(
|
|
Arg.Is<AuditEvent>(e =>
|
|
e.EventId == packet.Audit.EventId
|
|
&& e.AsRow().Kind == AuditKind.CachedResolve
|
|
&& e.AsRow().Status == AuditStatus.Delivered),
|
|
Arg.Any<CancellationToken>());
|
|
|
|
await _tracking.Received(1).RecordTerminalAsync(
|
|
_id, "Delivered", null, null, Arg.Any<CancellationToken>());
|
|
await _tracking.DidNotReceiveWithAnyArgs().RecordEnqueueAsync(
|
|
default, default!, default, default, default, default, default);
|
|
await _tracking.DidNotReceiveWithAnyArgs().RecordAttemptAsync(
|
|
default, default!, default, default, default, default);
|
|
}
|
|
|
|
// ── 2026-09-06: the audit row is enqueued, never awaited to commit ─────
|
|
// Design: docs/plans/2026-09-06-tracking-store-async-writes-design.md
|
|
|
|
/// <summary>Verifies that forwarding returns while the audit commit is still pending.</summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task ForwardAsync_ReturnsWhileTheAuditCommitIsStillPending()
|
|
{
|
|
// SqliteAuditWriter.WriteAsync returns the batch completion, which lands only
|
|
// after FlushIntervalMs + fsync. A never-completing task models "not flushed
|
|
// yet": ForwardAsync must still return and still record the tracking half.
|
|
var neverFlushed = new TaskCompletionSource();
|
|
_writer.WriteAsync(Arg.Any<AuditEvent>(), Arg.Any<CancellationToken>())
|
|
.Returns(neverFlushed.Task);
|
|
|
|
var sut = CreateSut();
|
|
var forward = sut.ForwardAsync(ResolvePacket(), CancellationToken.None);
|
|
|
|
var finished = await Task.WhenAny(forward, Task.Delay(TimeSpan.FromSeconds(2)));
|
|
Assert.Same(forward, finished);
|
|
await _writer.Received(1).WriteAsync(Arg.Any<AuditEvent>(), Arg.Any<CancellationToken>());
|
|
await _tracking.Received(1).RecordTerminalAsync(
|
|
_id, "Delivered", Arg.Any<string?>(), Arg.Any<int?>(), Arg.Any<CancellationToken>());
|
|
}
|
|
|
|
/// <summary>Verifies that a later audit commit fault is swallowed rather than surfaced to the caller.</summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task ForwardAsync_AuditCommitFaultsLater_IsSwallowed()
|
|
{
|
|
var lateFault = new TaskCompletionSource();
|
|
_writer.WriteAsync(Arg.Any<AuditEvent>(), Arg.Any<CancellationToken>())
|
|
.Returns(lateFault.Task);
|
|
|
|
var sut = CreateSut();
|
|
await sut.ForwardAsync(SubmitPacket(), CancellationToken.None); // returns before the fault
|
|
lateFault.SetException(new InvalidOperationException("disk full"));
|
|
await Task.Yield();
|
|
|
|
// Nothing propagates anywhere the script could see; the tracking half ran.
|
|
await _tracking.Received(1).RecordEnqueueAsync(
|
|
Arg.Any<TrackedOperationId>(), Arg.Any<string>(), Arg.Any<string?>(),
|
|
Arg.Any<string?>(), Arg.Any<string?>(), Arg.Any<string?>(), Arg.Any<CancellationToken>());
|
|
}
|
|
|
|
/// <summary>Verifies that a throwing audit writer is logged and the exception does not propagate.</summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task ForwardAsync_WriterThrows_Logs_DoesNotPropagate()
|
|
{
|
|
_writer.WriteAsync(Arg.Any<AuditEvent>(), Arg.Any<CancellationToken>())
|
|
.Throws(new InvalidOperationException("primary down"));
|
|
|
|
var sut = CreateSut();
|
|
|
|
// Must not throw.
|
|
await sut.ForwardAsync(SubmitPacket(), CancellationToken.None);
|
|
|
|
// Tracking still attempted — emission of the two halves is independent
|
|
// so a writer outage cannot starve the operational row (and vice-versa).
|
|
await _tracking.Received(1).RecordEnqueueAsync(
|
|
Arg.Any<TrackedOperationId>(),
|
|
Arg.Any<string>(),
|
|
Arg.Any<string?>(),
|
|
Arg.Any<string?>(),
|
|
Arg.Any<string?>(),
|
|
Arg.Any<string?>(),
|
|
Arg.Any<CancellationToken>());
|
|
}
|
|
|
|
/// <summary>Verifies that a throwing tracking store is logged and the exception does not propagate.</summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task ForwardAsync_TrackingStoreThrows_Logs_DoesNotPropagate()
|
|
{
|
|
_tracking.RecordEnqueueAsync(
|
|
Arg.Any<TrackedOperationId>(),
|
|
Arg.Any<string>(),
|
|
Arg.Any<string?>(),
|
|
Arg.Any<string?>(),
|
|
Arg.Any<string?>(),
|
|
Arg.Any<string?>(),
|
|
Arg.Any<CancellationToken>())
|
|
.Throws(new InvalidOperationException("sqlite locked"));
|
|
|
|
var sut = CreateSut();
|
|
|
|
await sut.ForwardAsync(SubmitPacket(), CancellationToken.None);
|
|
|
|
// Writer still attempted — emission halves are independent.
|
|
await _writer.Received(1).WriteAsync(
|
|
Arg.Any<AuditEvent>(), Arg.Any<CancellationToken>());
|
|
}
|
|
|
|
/// <summary>Verifies that forwarding a null packet throws.</summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task ForwardAsync_NullPacket_Throws()
|
|
{
|
|
var sut = CreateSut();
|
|
|
|
await Assert.ThrowsAsync<ArgumentNullException>(
|
|
() => sut.ForwardAsync(null!, CancellationToken.None));
|
|
}
|
|
|
|
// ── SourceNode stamping ──
|
|
|
|
/// <summary>Verifies that a submit packet is stamped with the source node from the node identity provider.</summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task ForwardAsync_Submit_StampsSourceNode_FromNodeIdentityProvider()
|
|
{
|
|
// SourceNode stamping: when an INodeIdentityProvider is
|
|
// wired the forwarder must stamp its NodeName onto the
|
|
// RecordEnqueueAsync sourceNode parameter so the tracking row
|
|
// captures the originating node (node-a/node-b).
|
|
var nodeIdentity = Substitute.For<INodeIdentityProvider>();
|
|
nodeIdentity.NodeName.Returns("node-a");
|
|
|
|
var sut = new CachedCallTelemetryForwarder(
|
|
_writer, _tracking, NullLogger<CachedCallTelemetryForwarder>.Instance, nodeIdentity);
|
|
|
|
await sut.ForwardAsync(SubmitPacket(), CancellationToken.None);
|
|
|
|
await _tracking.Received(1).RecordEnqueueAsync(
|
|
_id,
|
|
"ApiOutbound",
|
|
"ERP.GetOrder",
|
|
"inst-1",
|
|
"ScriptActor:doStuff",
|
|
"node-a",
|
|
Arg.Any<CancellationToken>());
|
|
}
|
|
|
|
/// <summary>Verifies that a node identity with a null node name results in a null source node.</summary>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
[Fact]
|
|
public async Task ForwardAsync_Submit_NodeIdentityNullNodeName_PassesNullSourceNode()
|
|
{
|
|
// The provider exists but reports a null NodeName (unconfigured).
|
|
// The forwarder passes that null through to RecordEnqueueAsync rather
|
|
// than falling back to a placeholder string.
|
|
var nodeIdentity = Substitute.For<INodeIdentityProvider>();
|
|
nodeIdentity.NodeName.Returns((string?)null);
|
|
|
|
var sut = new CachedCallTelemetryForwarder(
|
|
_writer, _tracking, NullLogger<CachedCallTelemetryForwarder>.Instance, nodeIdentity);
|
|
|
|
await sut.ForwardAsync(SubmitPacket(), CancellationToken.None);
|
|
|
|
await _tracking.Received(1).RecordEnqueueAsync(
|
|
_id,
|
|
"ApiOutbound",
|
|
"ERP.GetOrder",
|
|
"inst-1",
|
|
"ScriptActor:doStuff",
|
|
null,
|
|
Arg.Any<CancellationToken>());
|
|
}
|
|
}
|