diff --git a/docs/plans/2026-05-23-inbound-api-full-response-audit-design.md b/docs/plans/2026-05-23-inbound-api-full-response-audit-design.md index 81eb294..7cec6c6 100644 --- a/docs/plans/2026-05-23-inbound-api-full-response-audit-design.md +++ b/docs/plans/2026-05-23-inbound-api-full-response-audit-design.md @@ -19,9 +19,11 @@ receive" debugging path. For `Channel = ApiInbound` rows only, capture `RequestSummary` and `ResponseSummary` verbatim up to a hard per-body ceiling of **1 MB** (configurable). The 8 KB / 64 KB default/error caps that apply to other channels -do not apply here. All other channels (`ApiOutbound`, `DbOutbound`, -`Notification`, cached-call lifecycle, `InboundAuthFailure`) keep the existing -policy unchanged. +do not apply here. The carve-out is channel-scoped (NOT kind-scoped): every +`Channel = ApiInbound` row uses the inbound ceiling regardless of `Kind`, so +`InboundAuthFailure` rows pick up the same ceiling as `InboundRequest`. All +other channels (`ApiOutbound`, `DbOutbound`, `Notification`, cached-call +lifecycle) keep the existing policy unchanged. ## Capture Policy Change @@ -124,8 +126,9 @@ same "options validation" path used for other AuditLog settings. routinely-huge responses, operators use the existing per-target body redactor to compress them, or lower the global ceiling. - **Changes to other channels' caps.** `ApiOutbound`, `DbOutbound`, - `Notification`, cached-call lifecycle rows, and `InboundAuthFailure` keep the - existing 8 KB / 64 KB policy. + `Notification`, and cached-call lifecycle rows keep the existing 8 KB / 64 KB + policy. (`InboundAuthFailure` rows carry `Channel = ApiInbound` and so fall + under the inbound ceiling like every other inbound row.) ## Acceptance Criteria diff --git a/docs/requirements/Component-AuditLog.md b/docs/requirements/Component-AuditLog.md index 285f4f6..7280374 100644 --- a/docs/requirements/Component-AuditLog.md +++ b/docs/requirements/Component-AuditLog.md @@ -95,8 +95,8 @@ row per lifecycle event across all channels. | `DurationMs` | `int` NULL | Call / attempt duration. | | `ErrorMessage` | `nvarchar(1024)` NULL | Truncated; `ErrorDetail` for full text. | | `ErrorDetail` | `nvarchar(max)` NULL | Optional full exception text on failures. | -| `RequestSummary` | `nvarchar(max)` NULL | Truncated request payload (configurable cap). Headers redacted. | -| `ResponseSummary` | `nvarchar(max)` NULL | Truncated response payload. Full on errors. | +| `RequestSummary` | `nvarchar(max)` NULL | Truncated request payload (configurable cap). Headers redacted. For `Channel = ApiInbound`, captured in full up to `AuditLog:InboundMaxBytes` (default 1 MiB) — see Payload Capture Policy. | +| `ResponseSummary` | `nvarchar(max)` NULL | Truncated response payload. For `Channel = ApiInbound`, captured in full up to `AuditLog:InboundMaxBytes` (default 1 MiB). For other channels, capped at `DefaultCapBytes` by default and `ErrorCapBytes` on error rows. | | `PayloadTruncated` | `bit` | Set if either summary was truncated. | | `Extra` | `nvarchar(max)` NULL | Channel-specific JSON for fields we don't promote to columns. | @@ -262,6 +262,7 @@ operational `SiteCalls` shape for the dispatcher and UI. - **Default cap** — 8 KB for each of `RequestSummary` and `ResponseSummary`; raised to 64 KB on any error row (`Status IN ('Failed', 'Parked', 'Discarded')`). +- **Inbound API exception.** For `Channel = ApiInbound`, `RequestSummary` and `ResponseSummary` are captured in full up to a per-body hard ceiling of 1 MiB (configurable via `AuditLog:InboundMaxBytes`; default 1 048 576 bytes; min 8 192; max 16 777 216). The 8 KiB / 64 KiB default/error caps that apply to other channels do not apply here. `PayloadTruncated = 1` is set only when the inbound ceiling is hit — verbatim capture is the normal case. The ceiling applies independently to each body. Header redaction and per-target body redactors still run before persistence. - **Truncation** — UTF-8 byte-safe; `PayloadTruncated = 1` when applied. Full bodies are never stored. - **HTTP headers** — `Authorization`, `Cookie`, `Set-Cookie`, `X-API-Key`, and diff --git a/docs/requirements/Component-InboundAPI.md b/docs/requirements/Component-InboundAPI.md index 7844f1a..301afc8 100644 --- a/docs/requirements/Component-InboundAPI.md +++ b/docs/requirements/Component-InboundAPI.md @@ -116,7 +116,7 @@ API method scripts are compiled at central startup — all method definitions ar ## API Call Logging -- **Every request — success or failure — emits one `ApiInbound.Completed` row** to `ICentralAuditWriter` from request middleware before the HTTP response is flushed. The row captures the API key **name** (never the key material), remote IP, user-agent, response status, duration, and truncated request/response bodies per the Audit Log capture policy (see Component-AuditLog.md, Payload Capture Policy). This supersedes the earlier failures-only stance: operational API traffic is now part of the centralized audit log, so configuration changes and call activity share a single retention/query surface. +- **Every request — success or failure — emits one `ApiInbound.Completed` row** to `ICentralAuditWriter` from request middleware before the HTTP response is flushed. The row captures the API key **name** (never the key material), remote IP, user-agent, response status, duration, and the request/response bodies. Bodies are captured in full up to `AuditLog:InboundMaxBytes` (default 1 MiB); `PayloadTruncated = 1` only when that ceiling is hit. Header redaction and per-target body redactors still apply (see Component-AuditLog.md, Payload Capture Policy). This supersedes the earlier failures-only stance: operational API traffic is now part of the centralized audit log, so configuration changes and call activity share a single retention/query surface. - Script execution errors (500 responses) remain captured on the same `ApiInbound.Completed` row (response status + error fields) rather than emitting a separate failure-only event. - **Fail-soft semantics.** The audit write is synchronous (inline before the response is flushed), but failures are caught: a write that throws is logged and increments `CentralAuditWriteFailures` (see Health Monitoring #11) and the request still returns its normal HTTP response. A failed audit append never turns a successful API call into an error returned to the caller. - No rate limiting — this is a private API in a controlled industrial environment with a known set of callers. Misbehaving callers are handled operationally (disable the API key). @@ -199,7 +199,7 @@ Inbound API scripts **cannot** call shared scripts directly — shared scripts a - **Communication Layer**: Routes requests to sites when method implementations need site data. - **Security & Auth**: API key validation (separate from LDAP/AD — API uses key-based auth). - **Configuration Database (via IAuditService)**: All API key and method definition changes are audit logged. -- **Audit Log (#23)**: Every inbound API request emits an `ApiInbound.Completed` row via `ICentralAuditWriter` from request middleware (non-blocking for the HTTP response). Payload truncation/redaction follows the Audit Log Payload Capture Policy. +- **Audit Log (#23)**: Every inbound API request emits an `ApiInbound.Completed` row via `ICentralAuditWriter` from request middleware (non-blocking for the HTTP response). Request and response bodies are captured in full up to `AuditLog:InboundMaxBytes` (default 1 MiB) per the Audit Log Payload Capture Policy; redaction (headers + per-target body redactors) still applies before persistence. - **Cluster Infrastructure**: API is hosted on the active central node and fails over with it. ## Interactions diff --git a/src/ScadaLink.AuditLog/Configuration/AuditLogOptions.cs b/src/ScadaLink.AuditLog/Configuration/AuditLogOptions.cs index 89cfe9b..4860b1e 100644 --- a/src/ScadaLink.AuditLog/Configuration/AuditLogOptions.cs +++ b/src/ScadaLink.AuditLog/Configuration/AuditLogOptions.cs @@ -1,3 +1,6 @@ +using ScadaLink.Commons.Entities.Audit; +using ScadaLink.Commons.Types.Enums; + namespace ScadaLink.AuditLog.Configuration; /// @@ -33,4 +36,14 @@ public sealed class AuditLogOptions /// Central retention window in days (default 365, range [30, 3650]). public int RetentionDays { get; set; } = 365; + + /// + /// Per-body byte ceiling applied to and + /// for rows + /// (default 1 MiB). The 8 KiB / 64 KiB default/error caps that apply to other channels + /// do not apply here — inbound traffic captures verbatim up to this ceiling and only + /// then sets . See + /// docs/plans/2026-05-23-inbound-api-full-response-audit-design.md. + /// + public int InboundMaxBytes { get; set; } = 1_048_576; } diff --git a/src/ScadaLink.AuditLog/Configuration/AuditLogOptionsValidator.cs b/src/ScadaLink.AuditLog/Configuration/AuditLogOptionsValidator.cs index 59785c3..b67bd49 100644 --- a/src/ScadaLink.AuditLog/Configuration/AuditLogOptionsValidator.cs +++ b/src/ScadaLink.AuditLog/Configuration/AuditLogOptionsValidator.cs @@ -21,6 +21,12 @@ public sealed class AuditLogOptionsValidator : IValidateOptions /// Inclusive upper bound for . public const int MaxRetentionDays = 3650; + /// Inclusive lower bound for (8 KiB). + public const int MinInboundMaxBytes = 8_192; + + /// Inclusive upper bound for (16 MiB). + public const int MaxInboundMaxBytes = 16_777_216; + /// public ValidateOptionsResult Validate(string? name, AuditLogOptions options) { @@ -50,6 +56,13 @@ public sealed class AuditLogOptionsValidator : IValidateOptions $"must be in [{MinRetentionDays}, {MaxRetentionDays}] days."); } + if (options.InboundMaxBytes < MinInboundMaxBytes || options.InboundMaxBytes > MaxInboundMaxBytes) + { + failures.Add( + $"AuditLog:{nameof(AuditLogOptions.InboundMaxBytes)} ({options.InboundMaxBytes}) " + + $"must be in [{MinInboundMaxBytes}, {MaxInboundMaxBytes}] bytes."); + } + return failures.Count == 0 ? ValidateOptionsResult.Success : ValidateOptionsResult.Fail(failures); diff --git a/src/ScadaLink.AuditLog/Payload/DefaultAuditPayloadFilter.cs b/src/ScadaLink.AuditLog/Payload/DefaultAuditPayloadFilter.cs index 78328b1..0d22447 100644 --- a/src/ScadaLink.AuditLog/Payload/DefaultAuditPayloadFilter.cs +++ b/src/ScadaLink.AuditLog/Payload/DefaultAuditPayloadFilter.cs @@ -118,7 +118,14 @@ public sealed class DefaultAuditPayloadFilter : IAuditPayloadFilter try { var opts = _options.CurrentValue; - var cap = IsErrorStatus(rawEvent.Status) ? opts.ErrorCapBytes : opts.DefaultCapBytes; + // Inbound API gets a dedicated, larger ceiling — request/response bodies are + // captured verbatim up to InboundMaxBytes (default 1 MiB) so support can + // replay exactly what the caller sent and what we returned. Other channels + // keep the global 8 KiB / 64 KiB policy. + // See docs/plans/2026-05-23-inbound-api-full-response-audit-design.md. + var cap = rawEvent.Channel == AuditChannel.ApiInbound + ? opts.InboundMaxBytes + : (IsErrorStatus(rawEvent.Status) ? opts.ErrorCapBytes : opts.DefaultCapBytes); // --- Header-redaction stage (runs BEFORE truncation) ---------- var request = RedactHeaders(rawEvent.RequestSummary, opts.HeaderRedactList); diff --git a/src/ScadaLink.InboundAPI/Middleware/AuditWriteMiddleware.cs b/src/ScadaLink.InboundAPI/Middleware/AuditWriteMiddleware.cs index db83c85..66bf99c 100644 --- a/src/ScadaLink.InboundAPI/Middleware/AuditWriteMiddleware.cs +++ b/src/ScadaLink.InboundAPI/Middleware/AuditWriteMiddleware.cs @@ -1,8 +1,11 @@ +using System.Buffers; using System.Diagnostics; using System.Text; using System.Text.Json; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using ScadaLink.AuditLog.Configuration; using ScadaLink.Commons.Entities.Audit; using ScadaLink.Commons.Interfaces.Services; using ScadaLink.Commons.Types.Enums; @@ -42,11 +45,22 @@ namespace ScadaLink.InboundAPI.Middleware; /// /// Body capture. The request body is buffered via /// then -/// rewound so the downstream endpoint handler still sees the full payload. -/// Response body capture is deferred to M5 — wrapping Response.Body -/// requires a memory-stream swap that interacts awkwardly with Minimal API's -/// Results.Json/Results.Text writers; the M4 deliverable emits -/// the audit row with left null. +/// rewound so the downstream endpoint handler still sees the full payload. The +/// response body is captured by wrapping in a +/// forwarding stream that mirrors writes to the original sink (transparent to +/// the real client) while capturing a bounded copy for audit. +/// +/// +/// +/// Bounded capture at the source. Both the request- and response-body +/// audit copies are bounded at +/// (default 1 MiB) AT THE CAPTURE SITE — we never buffer more than +/// cap + 1 bytes per body even when the client streams hundreds of MiB. +/// The downstream handler and the real client still see every byte; only the +/// audit copy is bounded. The cap is also enforced again by +/// (which OR's +/// in its own determination), so a +/// row truncated here remains truncated even if the filter is bypassed. /// /// public sealed class AuditWriteMiddleware @@ -74,21 +88,29 @@ public sealed class AuditWriteMiddleware private readonly RequestDelegate _next; private readonly ICentralAuditWriter _auditWriter; private readonly ILogger _logger; + private readonly IOptionsMonitor _options; public AuditWriteMiddleware( RequestDelegate next, ICentralAuditWriter auditWriter, - ILogger logger) + ILogger logger, + IOptionsMonitor options) { _next = next ?? throw new ArgumentNullException(nameof(next)); _auditWriter = auditWriter ?? throw new ArgumentNullException(nameof(auditWriter)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _options = options ?? throw new ArgumentNullException(nameof(options)); } public async Task InvokeAsync(HttpContext ctx) { var sw = Stopwatch.StartNew(); + // Per-request hot read of the inbound cap — mirrors the convention used + // by DefaultAuditPayloadFilter so a live config change picks up on the + // next request without re-resolving the singleton. + var cap = _options.CurrentValue.InboundMaxBytes; + // Audit Log #23 (ParentExecutionId): mint the inbound request's per-request // ExecutionId ONCE, here at the start of the request, and stash it on // HttpContext.Items. Two consumers share this single id: @@ -106,7 +128,16 @@ public sealed class AuditWriteMiddleware // of the pipeline for us — but we also rewind to position 0 after our // own read so the very next reader starts from the top. ctx.Request.EnableBuffering(); - var requestBody = await ReadBufferedRequestBodyAsync(ctx.Request).ConfigureAwait(false); + var (requestBody, requestTruncated) = + await ReadBufferedRequestBodyAsync(ctx.Request, cap).ConfigureAwait(false); + + // Response body — wrap Response.Body in a forwarding stream that mirrors + // every write to the original sink (transparent to the real client) + // while capturing AT MOST `cap + 1` bytes for the audit copy. The + // original Response.Body is restored in the finally block. + var originalResponseBody = ctx.Response.Body; + using var captureStream = new CapturedResponseStream(originalResponseBody, cap); + ctx.Response.Body = captureStream; Exception? thrown = null; try @@ -123,7 +154,20 @@ public sealed class AuditWriteMiddleware finally { sw.Stop(); - EmitInboundAudit(ctx, sw.ElapsedMilliseconds, thrown, requestBody); + + // Restore the original stream and resolve the captured audit copy. + // The forwarding wrapper has already written every byte to the + // original sink; this just pulls back the bounded UTF-8 string. + ctx.Response.Body = originalResponseBody; + var (responseBody, responseTruncated) = captureStream.GetCapturedBody(); + + EmitInboundAudit( + ctx, + sw.ElapsedMilliseconds, + thrown, + requestBody, + responseBody, + requestTruncated || responseTruncated); } } @@ -136,7 +180,9 @@ public sealed class AuditWriteMiddleware HttpContext ctx, long durationMs, Exception? thrown, - string? requestBody) + string? requestBody, + string? responseBody, + bool payloadTruncated) { try { @@ -187,9 +233,8 @@ public sealed class AuditWriteMiddleware DurationMs = (int)Math.Min(durationMs, int.MaxValue), ErrorMessage = thrown?.Message, RequestSummary = requestBody, - // Response body capture is deferred to M5 (see XML doc above). - ResponseSummary = null, - PayloadTruncated = false, + ResponseSummary = responseBody, + PayloadTruncated = payloadTruncated, Extra = extra, // Central direct-write — no site-local forwarding state. ForwardState = null, @@ -210,39 +255,113 @@ public sealed class AuditWriteMiddleware } /// - /// Reads the buffered request body fully into a string and rewinds the - /// stream so the downstream handler sees the unconsumed payload. Returns - /// null for empty/missing bodies so the audit row's + /// Reads the buffered request body up to bytes + /// into a string for the audit copy and rewinds the stream so the + /// downstream handler sees the unconsumed payload. Returns + /// (null, false) for empty/missing bodies so the audit row's /// stays null rather than /// containing an empty string. /// - private static async Task ReadBufferedRequestBodyAsync(HttpRequest request) + /// + /// Reads AT MOST cap + 1 bytes from the request stream into a + /// scratch buffer; if the extra byte arrives the body is over the cap and + /// the returned string is UTF-8 byte-safe truncated to exactly + /// cap bytes with truncated = true. The cap applies only to + /// the audit copy — the request stream is always rewound to position 0 + /// afterwards so the framework's next reader (the endpoint handler's + /// JSON parser) sees the full body. + /// + private static async Task<(string? body, bool truncated)> ReadBufferedRequestBodyAsync( + HttpRequest request, + int capBytes) { if (request.ContentLength is 0) { - return null; + return (null, false); } + // Read AT MOST cap + 1 bytes — the extra byte tells us the body was + // over the cap without forcing us to allocate the whole payload. Rent + // the scratch buffer from the shared ArrayPool so we don't allocate + // (and immediately discard) `cap + 1` bytes per request — the pool + // may hand back a buffer LARGER than `limit`, so we treat `limit` + // (not `buffer.Length`) as the read ceiling. + var limit = capBytes + 1; + var buffer = ArrayPool.Shared.Rent(limit); try { request.Body.Position = 0; - using var reader = new StreamReader( - request.Body, - Encoding.UTF8, - detectEncodingFromByteOrderMarks: false, - bufferSize: 1024, - leaveOpen: true); - var content = await reader.ReadToEndAsync().ConfigureAwait(false); - request.Body.Position = 0; - return string.IsNullOrEmpty(content) ? null : content; + + var total = 0; + while (total < limit) + { + var read = await request.Body + .ReadAsync(buffer.AsMemory(total, limit - total)) + .ConfigureAwait(false); + if (read == 0) + { + break; + } + total += read; + } + + if (total == 0) + { + return (null, false); + } + + var truncated = total > capBytes; + var bytesForString = truncated ? capBytes : total; + var content = DecodeUtf8Bounded(buffer, bytesForString, cutAtValidBytes: truncated); + return (string.IsNullOrEmpty(content) ? null : content, truncated); } catch { // A failed body read must not abort the request — fall through // with a null RequestSummary; the audit row still records the // outcome. - return null; + return (null, false); } + finally + { + // Even on a thrown read, the downstream handler must see the full + // body from position 0 — never let a failed audit copy leak a + // truncated view. A rewind failure is swallowed: best-effort, + // same philosophy as the rest of the file. + try { request.Body.Position = 0; } catch { /* swallow */ } + ArrayPool.Shared.Return(buffer); + } + } + + /// + /// UTF-8 byte-safe decode of bytes from + /// . When is + /// true the input is the result of a hard byte-count truncation, so + /// we walk back from validBytes while the byte is a continuation + /// byte (byte & 0xC0 == 0x80) to avoid splitting a multi-byte + /// codepoint. When false the caller is decoding the full payload + /// and the boundary stands as-is. + /// + /// + /// Mirrors the algorithm in DefaultAuditPayloadFilter.TruncateUtf8; + /// kept local to avoid a backwards project reference from + /// ScadaLink.AuditLog into ScadaLink.InboundAPI. + /// + private static string DecodeUtf8Bounded(byte[] bytes, int validBytes, bool cutAtValidBytes) + { + if (validBytes <= 0) + { + return string.Empty; + } + var boundary = validBytes; + if (cutAtValidBytes) + { + while (boundary > 0 && (bytes[boundary] & 0xC0) == 0x80) + { + boundary--; + } + } + return Encoding.UTF8.GetString(bytes, 0, boundary); } /// @@ -321,4 +440,153 @@ public sealed class AuditWriteMiddleware return path[(lastSlash + 1)..]; } + + /// + /// Write-only forwarding wrapper that mirrors every + /// write to the inner ASP.NET (so the real + /// client receives all bytes) while capturing AT MOST cap + 1 bytes + /// into a private bounded for the audit copy. + /// + /// + /// + /// The inner sink is owned by the framework and is NOT disposed when this + /// wrapper is disposed — we only own the capture . + /// + /// + /// All Write overloads forward to the inner stream FIRST, then capture the + /// remaining quota. If the inner sink throws (e.g. the client disconnects), + /// the exception is allowed to propagate — capture is best-effort, the + /// real I/O is authoritative. The handler-throws-mid-response test + /// (ResponseBody_OnHandlerThrow_BodyCapturedUpToTheThrow) verifies + /// that captured bytes up to the throw are still recoverable. + /// + /// + private sealed class CapturedResponseStream : Stream + { + private readonly Stream _inner; + private readonly int _capBytes; + private readonly MemoryStream _captured; + private bool _disposed; + + public CapturedResponseStream(Stream inner, int capBytes) + { + _inner = inner ?? throw new ArgumentNullException(nameof(inner)); + _capBytes = Math.Max(0, capBytes); + // Capture up to cap + 1 bytes so we can detect the over-cap case + // without growing the buffer further. + _captured = new MemoryStream(); + } + + public override bool CanRead => false; + public override bool CanSeek => false; + public override bool CanWrite => true; + + public override long Length => + throw new NotSupportedException("CapturedResponseStream is write-only."); + + public override long Position + { + get => throw new NotSupportedException("CapturedResponseStream is write-only."); + set => throw new NotSupportedException("CapturedResponseStream is write-only."); + } + + public override void Flush() => _inner.Flush(); + + public override Task FlushAsync(CancellationToken cancellationToken) => + _inner.FlushAsync(cancellationToken); + + public override int Read(byte[] buffer, int offset, int count) => + throw new NotSupportedException("CapturedResponseStream is write-only."); + + public override long Seek(long offset, SeekOrigin origin) => + throw new NotSupportedException("CapturedResponseStream is write-only."); + + public override void SetLength(long value) => + throw new NotSupportedException("CapturedResponseStream is write-only."); + + public override void Write(byte[] buffer, int offset, int count) + { + // Forward to the real sink FIRST — the client must never miss + // bytes if capture throws. + _inner.Write(buffer, offset, count); + CaptureBytes(buffer.AsSpan(offset, count)); + } + + public override void Write(ReadOnlySpan buffer) + { + _inner.Write(buffer); + CaptureBytes(buffer); + } + + public override async Task WriteAsync( + byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + await _inner.WriteAsync(buffer.AsMemory(offset, count), cancellationToken) + .ConfigureAwait(false); + CaptureBytes(buffer.AsSpan(offset, count)); + } + + public override async ValueTask WriteAsync( + ReadOnlyMemory buffer, CancellationToken cancellationToken = default) + { + await _inner.WriteAsync(buffer, cancellationToken).ConfigureAwait(false); + CaptureBytes(buffer.Span); + } + + /// + /// Capture up to cap + 1 bytes total into the private + /// . Once the cap quota is reached, further + /// bytes are silently dropped from the audit copy (the real sink has + /// already received them upstream of this call). + /// + private void CaptureBytes(ReadOnlySpan span) + { + if (span.Length == 0) + { + return; + } + var quota = (_capBytes + 1) - (int)_captured.Length; + if (quota <= 0) + { + return; + } + var take = Math.Min(quota, span.Length); + _captured.Write(span[..take]); + } + + /// + /// Returns the captured response body as a UTF-8 string (byte-safe + /// truncated to cap bytes) and a flag indicating whether the + /// audit copy hit the cap. Returns (null, false) when no bytes + /// were captured, mirroring the request-body empty contract. + /// + public (string? body, bool truncated) GetCapturedBody() + { + var length = (int)_captured.Length; + if (length == 0) + { + return (null, false); + } + var truncated = length > _capBytes; + var bytes = _captured.GetBuffer(); + var bytesForString = truncated ? _capBytes : length; + var content = DecodeUtf8Bounded(bytes, bytesForString, cutAtValidBytes: truncated); + return (string.IsNullOrEmpty(content) ? null : content, truncated); + } + + protected override void Dispose(bool disposing) + { + if (!_disposed) + { + if (disposing) + { + // Own only the capture stream; the inner sink belongs to + // the framework's response pipeline. + _captured.Dispose(); + } + _disposed = true; + } + base.Dispose(disposing); + } + } } diff --git a/src/ScadaLink.InboundAPI/ScadaLink.InboundAPI.csproj b/src/ScadaLink.InboundAPI/ScadaLink.InboundAPI.csproj index 94cf9ec..7e898aa 100644 --- a/src/ScadaLink.InboundAPI/ScadaLink.InboundAPI.csproj +++ b/src/ScadaLink.InboundAPI/ScadaLink.InboundAPI.csproj @@ -14,6 +14,9 @@ + + diff --git a/tests/ScadaLink.AuditLog.Tests/Configuration/AuditLogOptionsBindingTests.cs b/tests/ScadaLink.AuditLog.Tests/Configuration/AuditLogOptionsBindingTests.cs index f9829cd..8cc07a7 100644 --- a/tests/ScadaLink.AuditLog.Tests/Configuration/AuditLogOptionsBindingTests.cs +++ b/tests/ScadaLink.AuditLog.Tests/Configuration/AuditLogOptionsBindingTests.cs @@ -45,7 +45,8 @@ public class AuditLogOptionsBindingTests "RedactSqlParamsMatching": "@token|@secret" } }, - "RetentionDays": 180 + "RetentionDays": 180, + "InboundMaxBytes": 524288 } } """; @@ -64,6 +65,7 @@ public class AuditLogOptionsBindingTests Assert.Equal(4096, opts.DefaultCapBytes); Assert.Equal(32768, opts.ErrorCapBytes); Assert.Equal(180, opts.RetentionDays); + Assert.Equal(524_288, opts.InboundMaxBytes); // HeaderRedactList: the Microsoft.Extensions.Configuration list binder // APPENDS to the default list, so we assert containment rather than diff --git a/tests/ScadaLink.AuditLog.Tests/Configuration/AuditLogOptionsValidatorTests.cs b/tests/ScadaLink.AuditLog.Tests/Configuration/AuditLogOptionsValidatorTests.cs new file mode 100644 index 0000000..6253746 --- /dev/null +++ b/tests/ScadaLink.AuditLog.Tests/Configuration/AuditLogOptionsValidatorTests.cs @@ -0,0 +1,53 @@ +using ScadaLink.AuditLog.Configuration; + +namespace ScadaLink.AuditLog.Tests.Configuration; + +/// +/// Task 1 of docs/plans/2026-05-23-inbound-api-full-response-audit.md: +/// pins the default to 1 MiB and +/// the validator bounds to [8 KiB, 16 MiB]. The inbound channel needs a +/// much larger ceiling than the 8 KiB / 64 KiB default/error caps that other +/// channels use, but unbounded would let any caller flood the central +/// AuditLog table with arbitrarily large bodies — hence the upper bound. +/// Companion to which covers the existing +/// cap-bytes + retention invariants. +/// +public class AuditLogOptionsValidatorTests +{ + [Fact] + public void Validate_InboundMaxBytes_DefaultOptions_IsOneMebibyte() + { + // The doc'd default per docs/plans/2026-05-23-inbound-api-full-response-audit-design.md + // is 1 048 576 bytes (1 MiB). Pin it so a config drift is a test failure, + // not a silent operational surprise. + var opts = new AuditLogOptions(); + Assert.Equal(1_048_576, opts.InboundMaxBytes); + } + + [Theory] + [InlineData(8_192)] // documented min + [InlineData(1_048_576)] // default + [InlineData(16_777_216)] // documented max + public void Validate_InboundMaxBytes_InRange_Passes(int value) + { + var validator = new AuditLogOptionsValidator(); + var opts = new AuditLogOptions { InboundMaxBytes = value }; + Assert.True(validator.Validate(null, opts).Succeeded); + } + + [Theory] + [InlineData(0)] + [InlineData(8_191)] + [InlineData(16_777_217)] + [InlineData(int.MaxValue)] + public void Validate_InboundMaxBytes_OutOfRange_Fails(int value) + { + var validator = new AuditLogOptionsValidator(); + var opts = new AuditLogOptions { InboundMaxBytes = value }; + var result = validator.Validate(null, opts); + Assert.False(result.Succeeded); + Assert.Contains( + result.Failures!, + f => f.Contains(nameof(AuditLogOptions.InboundMaxBytes), StringComparison.Ordinal)); + } +} diff --git a/tests/ScadaLink.AuditLog.Tests/Payload/InboundChannelCapTests.cs b/tests/ScadaLink.AuditLog.Tests/Payload/InboundChannelCapTests.cs new file mode 100644 index 0000000..42f5cf8 --- /dev/null +++ b/tests/ScadaLink.AuditLog.Tests/Payload/InboundChannelCapTests.cs @@ -0,0 +1,133 @@ +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using ScadaLink.AuditLog.Configuration; +using ScadaLink.AuditLog.Payload; +using ScadaLink.Commons.Entities.Audit; +using ScadaLink.Commons.Types.Enums; + +namespace ScadaLink.AuditLog.Tests.Payload; + +/// +/// Pins the docs/plans/2026-05-23-inbound-api-full-response-audit-design.md +/// inbound carve-out: ApiInbound rows use InboundMaxBytes (default 1 MiB) for +/// RequestSummary / ResponseSummary truncation, NOT DefaultCapBytes / +/// ErrorCapBytes. Other channels keep the existing caps. +/// +/// +/// Uses a file-local helper mirroring the +/// convention in the sibling Payload tests (TruncationTests, +/// FilterIntegrationTests, BodyRegexRedactionTests, etc.) — the +/// TestOptionsMonitor<T> helper referenced by the plan is a +/// private nested class inside AuditLogOptionsBindingTests and thus +/// not reachable from this file. +/// +public class InboundChannelCapTests +{ + private static AuditEvent MakeInbound( + AuditStatus status, + string? request = null, + string? response = null) => + new() + { + EventId = Guid.NewGuid(), + OccurredAtUtc = DateTime.UtcNow, + Channel = AuditChannel.ApiInbound, + Kind = AuditKind.InboundRequest, + Status = status, + RequestSummary = request, + ResponseSummary = response, + }; + + [Fact] + public void ApiInbound_Delivered_RequestBody_BelowInboundMaxBytes_NotTruncated() + { + // Body well above the legacy 8 KiB default cap but under the 1 MiB + // inbound ceiling — must NOT truncate. + var body = new string('a', 100_000); + var opts = new AuditLogOptions(); // defaults + var filter = new DefaultAuditPayloadFilter( + new StaticMonitor(opts), + NullLogger.Instance); + + var result = filter.Apply(MakeInbound(AuditStatus.Delivered, request: body)); + + Assert.False(result.PayloadTruncated); + Assert.Equal(100_000, Encoding.UTF8.GetByteCount(result.RequestSummary!)); + } + + [Fact] + public void ApiInbound_Delivered_ResponseBody_BelowInboundMaxBytes_NotTruncated() + { + var body = new string('a', 100_000); + var opts = new AuditLogOptions(); + var filter = new DefaultAuditPayloadFilter( + new StaticMonitor(opts), + NullLogger.Instance); + + var result = filter.Apply(MakeInbound(AuditStatus.Delivered, response: body)); + + Assert.False(result.PayloadTruncated); + Assert.Equal(100_000, Encoding.UTF8.GetByteCount(result.ResponseSummary!)); + } + + [Fact] + public void ApiInbound_Failed_BodyAboveInboundMaxBytes_TruncatedToInboundMaxBytes() + { + // Even on error rows, the inbound cap is InboundMaxBytes (NOT ErrorCapBytes). + var opts = new AuditLogOptions { InboundMaxBytes = 16_384 }; + var oversized = new string('z', 50_000); + var filter = new DefaultAuditPayloadFilter( + new StaticMonitor(opts), + NullLogger.Instance); + + var result = filter.Apply(MakeInbound(AuditStatus.Failed, response: oversized)); + + Assert.True(result.PayloadTruncated); + Assert.True(Encoding.UTF8.GetByteCount(result.ResponseSummary!) <= 16_384); + } + + [Fact] + public void ApiOutbound_StillUsesDefaultCap_NotInboundMaxBytes() + { + // Regression guard: lifting the inbound cap MUST NOT change other + // channels. An ApiOutbound 100 KB body still hits the 8 KiB cap. + var opts = new AuditLogOptions(); + var body = new string('a', 100_000); + var filter = new DefaultAuditPayloadFilter( + new StaticMonitor(opts), + NullLogger.Instance); + + var evt = new AuditEvent + { + EventId = Guid.NewGuid(), + OccurredAtUtc = DateTime.UtcNow, + Channel = AuditChannel.ApiOutbound, + Kind = AuditKind.ApiCall, + Status = AuditStatus.Delivered, + RequestSummary = body, + }; + var result = filter.Apply(evt); + + Assert.True(result.PayloadTruncated); + Assert.True(Encoding.UTF8.GetByteCount(result.RequestSummary!) <= opts.DefaultCapBytes); + } + + /// + /// IOptionsMonitor test double — returns the same snapshot on every read, + /// no change-token plumbing required for these tests. Mirrors the helper + /// used in TruncationTests, FilterIntegrationTests, etc. + /// + private sealed class StaticMonitor : IOptionsMonitor + { + private readonly AuditLogOptions _value; + + public StaticMonitor(AuditLogOptions value) => _value = value; + + public AuditLogOptions CurrentValue => _value; + + public AuditLogOptions Get(string? name) => _value; + + public IDisposable? OnChange(Action listener) => null; + } +} diff --git a/tests/ScadaLink.InboundAPI.Tests/Middleware/AuditWriteMiddlewareTests.cs b/tests/ScadaLink.InboundAPI.Tests/Middleware/AuditWriteMiddlewareTests.cs index a28a6f6..cd220a8 100644 --- a/tests/ScadaLink.InboundAPI.Tests/Middleware/AuditWriteMiddlewareTests.cs +++ b/tests/ScadaLink.InboundAPI.Tests/Middleware/AuditWriteMiddlewareTests.cs @@ -4,6 +4,8 @@ using System.Text.Json; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Routing; using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using ScadaLink.AuditLog.Configuration; using ScadaLink.Commons.Entities.Audit; using ScadaLink.Commons.Interfaces.Services; using ScadaLink.Commons.Types.Enums; @@ -79,8 +81,32 @@ public class AuditWriteMiddlewareTests private static AuditWriteMiddleware CreateMiddleware( RequestDelegate next, - ICentralAuditWriter writer) => - new(next, writer, NullLogger.Instance); + ICentralAuditWriter writer, + AuditLogOptions? options = null) => + new( + next, + writer, + NullLogger.Instance, + new StaticAuditLogOptionsMonitor(options ?? new AuditLogOptions())); + + /// + /// File-local test double — returns the + /// same snapshot on every read, no change-token plumbing required. Mirrors the + /// StaticMonitor pattern in + /// tests/ScadaLink.AuditLog.Tests/Payload/InboundChannelCapTests.cs. + /// + private sealed class StaticAuditLogOptionsMonitor : IOptionsMonitor + { + private readonly AuditLogOptions _value; + + public StaticAuditLogOptionsMonitor(AuditLogOptions value) => _value = value; + + public AuditLogOptions CurrentValue => _value; + + public AuditLogOptions Get(string? name) => _value; + + public IDisposable? OnChange(Action listener) => null; + } // --------------------------------------------------------------------- // 1. Happy path — InboundRequest/Delivered/HttpStatus 200 @@ -487,4 +513,180 @@ public class AuditWriteMiddlewareTests Assert.NotNull(evt.DurationMs); Assert.True(evt.DurationMs >= 0); } + + // --------------------------------------------------------------------- + // Response body capture — Audit Log #23 (inbound full-response feature). + // Until the M5-deferred work landed, ResponseSummary was always null. + // These tests pin the new contract: the middleware wraps Response.Body, + // runs the pipeline, copies the buffered bytes back to the real stream, + // and stashes a UTF-8 string copy on ResponseSummary. + // --------------------------------------------------------------------- + + [Fact] + public async Task ResponseBody_IsCaptured_OnResponseSummary() + { + var writer = new RecordingAuditWriter(); + var ctx = BuildContext(); + var responseJson = "{\"result\":42}"; + var mw = CreateMiddleware(async hc => + { + hc.Response.StatusCode = 200; + hc.Response.ContentType = "application/json"; + await hc.Response.WriteAsync(responseJson); + }, writer); + + await mw.InvokeAsync(ctx); + + var evt = Assert.Single(writer.Events); + Assert.Equal(responseJson, evt.ResponseSummary); + } + + [Fact] + public async Task ResponseBody_IsForwardedToOriginalStream_DownstreamReadersSeeIt() + { + // Wrapping the response body must be TRANSPARENT — the real client + // stream still receives every byte the pipeline wrote. + var writer = new RecordingAuditWriter(); + var ctx = BuildContext(); + var captured = new MemoryStream(); + ctx.Response.Body = captured; // simulate the client/test sink + + var responseJson = "{\"ok\":true}"; + var mw = CreateMiddleware(async hc => + { + hc.Response.StatusCode = 200; + await hc.Response.WriteAsync(responseJson); + }, writer); + + await mw.InvokeAsync(ctx); + + Assert.Equal(responseJson, Encoding.UTF8.GetString(captured.ToArray())); + } + + [Fact] + public async Task ResponseBody_Empty_LeavesResponseSummaryNull() + { + // No bytes written => null, not empty-string. Mirrors the request-body + // contract in ReadBufferedRequestBodyAsync. + var writer = new RecordingAuditWriter(); + var ctx = BuildContext(); + var mw = CreateMiddleware(hc => + { + hc.Response.StatusCode = 204; + return Task.CompletedTask; + }, writer); + + await mw.InvokeAsync(ctx); + + var evt = Assert.Single(writer.Events); + Assert.Null(evt.ResponseSummary); + Assert.Equal(204, evt.HttpStatus); + } + + [Fact] + public async Task ResponseBody_OnHandlerThrow_BodyCapturedUpToTheThrow() + { + // If the handler writes some bytes then throws, the audit row still + // surfaces whatever the framework had flushed. The middleware re-throws + // (audit is best-effort, the request's error path stays authoritative). + var writer = new RecordingAuditWriter(); + var ctx = BuildContext(); + var boom = new InvalidOperationException("kaboom"); + var mw = CreateMiddleware(async hc => + { + hc.Response.StatusCode = 500; + await hc.Response.WriteAsync("partial"); + throw boom; + }, writer); + + var thrown = await Assert.ThrowsAsync( + () => mw.InvokeAsync(ctx)); + Assert.Same(boom, thrown); + + var evt = Assert.Single(writer.Events); + Assert.Equal(AuditStatus.Failed, evt.Status); + Assert.Equal("partial", evt.ResponseSummary); + } + + // --------------------------------------------------------------------- + // Bounded audit capture — memory safety follow-up. The capture site now + // honours AuditLogOptions.InboundMaxBytes at READ time (not just at + // filter-time), so a 500 MiB body cannot transiently allocate 500 MiB of + // string. The cap is local to the AUDIT copy; downstream readers and the + // real client still see every byte. + // --------------------------------------------------------------------- + + [Fact] + public async Task RequestBody_AboveInboundMaxBytes_TruncatedToCap_PayloadTruncatedTrue() + { + // 4 KiB cap, 20 KB body — the audit copy must be UTF-8 byte-safe + // capped at 4 KiB AND PayloadTruncated must flip, while the + // downstream handler still sees the full 20 KB payload. + const int cap = 4096; + var bigBody = new string('a', 20_000); + var writer = new RecordingAuditWriter(); + var ctx = BuildContext(body: bigBody); + + string? observedAfterMiddleware = null; + var mw = CreateMiddleware( + async hc => + { + using var reader = new StreamReader(hc.Request.Body); + observedAfterMiddleware = await reader.ReadToEndAsync(); + hc.Response.StatusCode = 200; + }, + writer, + options: new AuditLogOptions { InboundMaxBytes = cap }); + + await mw.InvokeAsync(ctx); + + // (iii) Downstream handler still sees the FULL body — the cap applied + // only to the audit copy. + Assert.Equal(bigBody, observedAfterMiddleware); + + var evt = Assert.Single(writer.Events); + // (i) Audit copy bounded at cap bytes (UTF-8 byte count). + Assert.NotNull(evt.RequestSummary); + Assert.True( + Encoding.UTF8.GetByteCount(evt.RequestSummary!) <= cap, + $"RequestSummary byte count {Encoding.UTF8.GetByteCount(evt.RequestSummary!)} exceeded cap {cap}"); + // (ii) Truncation flag set by the middleware (the filter will OR its + // own determination on top, but the middleware MUST set it itself). + Assert.True(evt.PayloadTruncated); + } + + [Fact] + public async Task ResponseBody_AboveInboundMaxBytes_TruncatedToCap_ClientStillReceivesAllBytes_PayloadTruncatedTrue() + { + // 4 KiB cap, 20 KB response — the test sink (acts as the real client) + // MUST receive all 20 KB while the audit copy is bounded at 4 KiB. + const int cap = 4096; + var bigResponse = new string('b', 20_000); + var writer = new RecordingAuditWriter(); + var ctx = BuildContext(); + var captured = new MemoryStream(); + ctx.Response.Body = captured; // stand-in for the client sink + + var mw = CreateMiddleware( + async hc => + { + hc.Response.StatusCode = 200; + await hc.Response.WriteAsync(bigResponse); + }, + writer, + options: new AuditLogOptions { InboundMaxBytes = cap }); + + await mw.InvokeAsync(ctx); + + // Client sink received every byte — the forwarding wrap is transparent. + Assert.Equal(bigResponse, Encoding.UTF8.GetString(captured.ToArray())); + + var evt = Assert.Single(writer.Events); + // Audit copy bounded at cap bytes. + Assert.NotNull(evt.ResponseSummary); + Assert.True( + Encoding.UTF8.GetByteCount(evt.ResponseSummary!) <= cap, + $"ResponseSummary byte count {Encoding.UTF8.GetByteCount(evt.ResponseSummary!)} exceeded cap {cap}"); + Assert.True(evt.PayloadTruncated); + } } diff --git a/tests/ScadaLink.InboundAPI.Tests/Middleware/MiddlewareOrderTests.cs b/tests/ScadaLink.InboundAPI.Tests/Middleware/MiddlewareOrderTests.cs index b74356f..f38dec2 100644 --- a/tests/ScadaLink.InboundAPI.Tests/Middleware/MiddlewareOrderTests.cs +++ b/tests/ScadaLink.InboundAPI.Tests/Middleware/MiddlewareOrderTests.cs @@ -6,6 +6,8 @@ using Microsoft.AspNetCore.TestHost; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using ScadaLink.AuditLog.Configuration; using ScadaLink.Commons.Entities.Audit; using ScadaLink.Commons.Interfaces.Services; using ScadaLink.Commons.Types.Enums; @@ -145,7 +147,8 @@ public class MiddlewareOrderTests // instantiates the type correctly. _ => Task.CompletedTask, writer, - NullLogger.Instance)); + NullLogger.Instance, + new StaticAuditLogOptionsMonitor(new AuditLogOptions()))); services.AddRouting(); services.AddAuthorization(); services.AddAuthentication("TestScheme") @@ -233,4 +236,22 @@ public class MiddlewareOrderTests return Task.FromResult(AuthenticateResult.Success(ticket)); } } + + /// + /// File-local test double — returns the + /// same snapshot on every read. Mirrors the helper in + /// AuditWriteMiddlewareTests. + /// + private sealed class StaticAuditLogOptionsMonitor : IOptionsMonitor + { + private readonly AuditLogOptions _value; + + public StaticAuditLogOptionsMonitor(AuditLogOptions value) => _value = value; + + public AuditLogOptions CurrentValue => _value; + + public AuditLogOptions Get(string? name) => _value; + + public IDisposable? OnChange(Action listener) => null; + } }