diff --git a/docs/plans/2026-05-20-auditlog-m5-payload-redaction.md b/docs/plans/2026-05-20-auditlog-m5-payload-redaction.md new file mode 100644 index 0000000..28f615e --- /dev/null +++ b/docs/plans/2026-05-20-auditlog-m5-payload-redaction.md @@ -0,0 +1,20 @@ +# Audit Log #23 — M5 Payload + Redaction Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers-extended-cc:subagent-driven-development (bundled cadence). + +**Goal:** Filter pipeline (IAuditPayloadFilter) runs between event construction and writer call. Truncates to 8 KB / 64 KB on error; applies HTTP header redactors (default list from M1-T9 AuditLogOptions); applies body regex redactors (global + per-target); applies SQL parameter redactors (per-connection opt-in); over-redacts on regex error and increments AuditRedactionFailure metric. Hot-reloadable config via IOptionsMonitor. + +**Vocabulary (M1 reality):** Error-row cap (64 KB) triggers when `Status NOT IN (AuditStatus.Delivered, AuditStatus.Submitted, AuditStatus.Forwarded)` — i.e., on `Failed/Parked/Discarded/Attempted/Skipped`. The roadmap's M5-T2 step references (Status=TransientFailure/PermanentFailure) are stale pre-M1 wording. Translation: `TransientFailure` = `Attempted` with HttpStatus 5xx OR `Failed`; `PermanentFailure` = `Failed`. + +**M4 realities baked in:** AuditingDb decorators, NotificationOutboxActor, AuditWriteMiddleware, site emission paths all need filter pluggin. Filter is invoked in: +- FallbackAuditWriter.WriteAsync (site chain) — before SqliteAuditWriter.WriteAsync. +- CentralAuditWriter.WriteAsync (central direct-write) — before IAuditLogRepository.InsertIfNotExistsAsync. +- AuditLogIngestActor handlers — before InsertIfNotExistsAsync/UpsertAsync. + +**Bundles:** +- Bundle A — Filter contract + truncation (T1, T2). +- Bundle B — Header + body + SQL param redaction (T3, T4, T5). +- Bundle C — Wire into emission paths + health metric (T6, T7). +- Bundle D — Configuration binding + perf + safety-net edge cases (T8, T9, T10). + +Final cross-bundle review + merge. diff --git a/src/ScadaLink.AuditLog/Central/AuditLogIngestActor.cs b/src/ScadaLink.AuditLog/Central/AuditLogIngestActor.cs index 2b2c580..8e7f21b 100644 --- a/src/ScadaLink.AuditLog/Central/AuditLogIngestActor.cs +++ b/src/ScadaLink.AuditLog/Central/AuditLogIngestActor.cs @@ -1,6 +1,7 @@ using Akka.Actor; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using ScadaLink.AuditLog.Payload; using ScadaLink.Commons.Entities.Audit; using ScadaLink.Commons.Interfaces.Repositories; using ScadaLink.Commons.Messages.Audit; @@ -114,8 +115,15 @@ public class AuditLogIngestActor : ReceiveActor // Resolve the repository for the whole batch — one DbContext per // message, mirroring NotificationOutboxActor. The injected-repository // mode (Bundle D tests) skips the scope entirely. + // Bundle C (M5-T6): the IAuditPayloadFilter is also resolved from the + // per-message scope when one is available so the row is truncated + + // redacted before InsertIfNotExistsAsync. The single-repository test + // ctor has no service provider — it falls through with no filter, + // which preserves the small-payload assumptions baked into the + // existing D2 fixtures. IServiceScope? scope = null; IAuditLogRepository repository; + IAuditPayloadFilter? filter = null; if (_injectedRepository is not null) { repository = _injectedRepository; @@ -124,6 +132,7 @@ public class AuditLogIngestActor : ReceiveActor { scope = _serviceProvider!.CreateScope(); repository = scope.ServiceProvider.GetRequiredService(); + filter = scope.ServiceProvider.GetService(); } try @@ -136,7 +145,11 @@ public class AuditLogIngestActor : ReceiveActor // repository hardening already swallows duplicate-key races, // so the same id arriving twice (site retry, reconciliation) // is a silent no-op. - var ingested = evt with { IngestedAtUtc = nowUtc }; + // Filter BEFORE the IngestedAtUtc stamp so the redacted + // copy carries the central-side ingest timestamp. Filter + // is contract-bound to never throw; null = pass-through. + var filtered = filter?.Apply(evt) ?? evt; + var ingested = filtered with { IngestedAtUtc = nowUtc }; await repository.InsertIfNotExistsAsync(ingested).ConfigureAwait(false); accepted.Add(evt.EventId); } @@ -185,6 +198,12 @@ public class AuditLogIngestActor : ReceiveActor var auditRepo = scope.ServiceProvider.GetRequiredService(); var siteCallRepo = scope.ServiceProvider.GetRequiredService(); var dbContext = scope.ServiceProvider.GetRequiredService(); + // Bundle C (M5-T6): resolve the filter for the whole batch from + // the scope; null = pass-through for test composition roots that + // skip the filter registration. The filter is contract-bound to + // never throw, so we can apply it inside the per-entry try + // without risking an unbounded blast radius. + var filter = scope.ServiceProvider.GetService(); foreach (var entry in cmd.Entries) { @@ -199,7 +218,12 @@ public class AuditLogIngestActor : ReceiveActor // matching timestamps (debugging convenience, not a // correctness invariant). var ingestedAt = DateTime.UtcNow; - var auditStamped = entry.Audit with { IngestedAtUtc = ingestedAt }; + // Filter the audit half BEFORE the dual-write — only the + // AuditLog row's payload columns are filterable; SiteCalls + // carries operational state only (status, retry count) and + // is left untouched. + var filteredAudit = filter?.Apply(entry.Audit) ?? entry.Audit; + var auditStamped = filteredAudit with { IngestedAtUtc = ingestedAt }; var siteCallStamped = entry.SiteCall with { IngestedAtUtc = ingestedAt }; await auditRepo.InsertIfNotExistsAsync(auditStamped) diff --git a/src/ScadaLink.AuditLog/Central/CentralAuditWriter.cs b/src/ScadaLink.AuditLog/Central/CentralAuditWriter.cs index fd0972d..ff48bea 100644 --- a/src/ScadaLink.AuditLog/Central/CentralAuditWriter.cs +++ b/src/ScadaLink.AuditLog/Central/CentralAuditWriter.cs @@ -1,5 +1,6 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using ScadaLink.AuditLog.Payload; using ScadaLink.Commons.Entities.Audit; using ScadaLink.Commons.Interfaces.Repositories; using ScadaLink.Commons.Interfaces.Services; @@ -40,11 +41,24 @@ public sealed class CentralAuditWriter : ICentralAuditWriter { private readonly IServiceProvider _services; private readonly ILogger _logger; + private readonly IAuditPayloadFilter? _filter; - public CentralAuditWriter(IServiceProvider services, ILogger logger) + /// + /// Bundle C (M5-T6) — the central direct-write path used by the + /// NotificationOutboxActor dispatch and the Inbound API middleware also + /// needs to truncate + redact before the row hits MS SQL. The filter is + /// optional so the M4 test composition roots that don't pass one keep + /// working (they only ever write small payloads); production DI registers + /// the real filter via . + /// + public CentralAuditWriter( + IServiceProvider services, + ILogger logger, + IAuditPayloadFilter? filter = null) { _services = services ?? throw new ArgumentNullException(nameof(services)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _filter = filter; } /// @@ -65,9 +79,14 @@ public sealed class CentralAuditWriter : ICentralAuditWriter try { + // Filter BEFORE stamping IngestedAtUtc + handing to the repo. The + // filter contract is "never throws"; the null-coalesce keeps the + // M4 test composition roots (no filter passed) working unchanged. + var filtered = _filter?.Apply(evt) ?? evt; + await using var scope = _services.CreateAsyncScope(); var repo = scope.ServiceProvider.GetRequiredService(); - var stamped = evt with { IngestedAtUtc = DateTime.UtcNow }; + var stamped = filtered with { IngestedAtUtc = DateTime.UtcNow }; await repo.InsertIfNotExistsAsync(stamped, ct).ConfigureAwait(false); } catch (Exception ex) diff --git a/src/ScadaLink.AuditLog/Configuration/PerTargetRedactionOverride.cs b/src/ScadaLink.AuditLog/Configuration/PerTargetRedactionOverride.cs index 739ee07..0673a4b 100644 --- a/src/ScadaLink.AuditLog/Configuration/PerTargetRedactionOverride.cs +++ b/src/ScadaLink.AuditLog/Configuration/PerTargetRedactionOverride.cs @@ -14,4 +14,15 @@ public sealed class PerTargetRedactionOverride /// Additional body redactor regex patterns (appended to the global list). public List? AdditionalBodyRedactors { get; set; } + + /// + /// Opt-in SQL parameter redaction: case-insensitive regex matched against + /// each SQL parameter NAME in the M4 AuditingDbCommand RequestSummary + /// JSON ({"sql":"...","parameters":{"@name":"value", ...}}); values + /// whose name matches are replaced with <redacted>. Null (the + /// default) means parameter values are captured verbatim. Only applied to + /// + /// rows. + /// + public string? RedactSqlParamsMatching { get; set; } } diff --git a/src/ScadaLink.AuditLog/Payload/DefaultAuditPayloadFilter.cs b/src/ScadaLink.AuditLog/Payload/DefaultAuditPayloadFilter.cs new file mode 100644 index 0000000..78328b1 --- /dev/null +++ b/src/ScadaLink.AuditLog/Payload/DefaultAuditPayloadFilter.cs @@ -0,0 +1,573 @@ +using System.Collections.Concurrent; +using System.Text; +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using ScadaLink.AuditLog.Configuration; +using ScadaLink.Commons.Entities.Audit; +using ScadaLink.Commons.Types.Enums; + +namespace ScadaLink.AuditLog.Payload; + +/// +/// Default . Bundle A established the +/// truncation backbone; Bundle B chains HTTP header redaction (M5-T3) BEFORE +/// truncation so redactors operate on the full payload and the cap then trims +/// the redacted result. +/// +/// +/// +/// Uses (not ) +/// so the M5-T8 hot-reload path sees fresh values without re-resolving the +/// singleton. reads +/// on every call, and the regex cache is keyed by pattern string — patterns +/// added via a live config change compile on first use of the next event; +/// patterns removed simply stop being looked up. No OnChange subscription +/// or explicit cache invalidation is required (the +/// AuditLogOptionsBindingTests fixture in ScadaLink.AuditLog.Tests +/// pins this behaviour). +/// +/// +/// "Error row" = NOT IN (Delivered, +/// Submitted, Forwarded) — every other status, including the +/// non-terminal Attempted, the parked/discarded terminals, and the +/// short-circuit Skipped, receives the larger error cap so a verbose +/// error body survives. +/// +/// +/// Apply MUST NOT throw — on internal failure the filter over-redacts by +/// returning the input with set and +/// increments the AuditRedactionFailure health metric via the injected +/// . Each redactor stage runs in +/// its own try/catch — a failure in (say) the header redactor still lets the +/// SQL parameter redactor and the truncator run on the remaining fields. +/// +/// +/// Stage order (each runs on every applicable field): +/// header redaction → body regex redaction → truncation. The SQL-parameter +/// stage piggybacks on the body-redactor path; both run BEFORE truncation so +/// the cap trims the redacted result, never bytes the redactor intended to +/// hide. +/// +/// +public sealed class DefaultAuditPayloadFilter : IAuditPayloadFilter +{ + private const string RedactedMarker = ""; + private const string RedactorErrorMarker = ""; + + /// + /// Per-match regex timeout. Catastrophic-backtracking patterns trip a + /// when a single match takes + /// longer than this; the offending field is then over-redacted with + /// and the failure counter is bumped. + /// 50 ms is generous for normal patterns yet short enough that the + /// audit hot-path isn't held up by a misconfigured regex. + /// + private static readonly TimeSpan RegexMatchTimeout = TimeSpan.FromMilliseconds(50); + + /// + /// JSON serializer options used to re-emit redacted summaries. The + /// UnsafeRelaxedJsonEscaping encoder is required so the redaction marker + /// (which contains < / >) survives unescaped — the + /// header-redaction tests grep for the literal marker, and the downstream + /// UI / log readers would rather see <redacted> than + /// . The summaries are persisted to the audit + /// table and rendered in trusted-internal contexts only, so the relaxed + /// HTML-escaping rules do not introduce an XSS surface. + /// + private static readonly JsonSerializerOptions RedactedSummaryJsonOptions = new() + { + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }; + + private readonly IOptionsMonitor _options; + private readonly ILogger _logger; + private readonly IAuditRedactionFailureCounter _failureCounter; + + /// + /// Compiled-regex cache keyed by pattern string. Lazy population: each + /// pattern is compiled on first use and cached forever (the entry's + /// carries either the working + /// or a sentinel marking the pattern as invalid so we don't retry the + /// failing compile on every call). ConcurrentDictionary is the right + /// thread-safety primitive here because the filter is a DI singleton + /// shared across the audit hot-path. + /// + private readonly ConcurrentDictionary _regexCache = new(); + + /// + /// Primary constructor used by DI — pulls the optional redaction-failure + /// counter from the container; a NoOp default is registered in + /// . + /// + public DefaultAuditPayloadFilter( + IOptionsMonitor options, + ILogger logger, + IAuditRedactionFailureCounter? failureCounter = null) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _failureCounter = failureCounter ?? new NoOpAuditRedactionFailureCounter(); + } + + public AuditEvent Apply(AuditEvent rawEvent) + { + try + { + var opts = _options.CurrentValue; + var cap = IsErrorStatus(rawEvent.Status) ? opts.ErrorCapBytes : opts.DefaultCapBytes; + + // --- Header-redaction stage (runs BEFORE truncation) ---------- + var request = RedactHeaders(rawEvent.RequestSummary, opts.HeaderRedactList); + var response = RedactHeaders(rawEvent.ResponseSummary, opts.HeaderRedactList); + var errorDetail = rawEvent.ErrorDetail; + var extra = rawEvent.Extra; + + // --- Body-regex stage (also runs BEFORE truncation) ----------- + // Resolves the active regex set per event so per-target overrides + // bound to AuditEvent.Target are picked up; effectively a no-op + // when neither GlobalBodyRedactors nor the per-target additions + // are configured. + var bodyRegexes = ResolveBodyRegexes(opts, rawEvent.Target); + if (bodyRegexes.Count > 0) + { + request = RedactBody(request, bodyRegexes); + response = RedactBody(response, bodyRegexes); + errorDetail = RedactBody(errorDetail, bodyRegexes); + extra = RedactBody(extra, bodyRegexes); + } + + // --- SQL parameter redaction stage (DbOutbound only) ---------- + // Parses the M4 AuditingDbCommand RequestSummary shape + // {"sql":"...","parameters":{...}} and redacts parameter VALUES + // whose NAME matches the per-connection regex. Opt-in: no + // PerTargetOverrides[connectionName].RedactSqlParamsMatching => + // no-op. Channel-guarded so the same regex can never accidentally + // touch an ApiOutbound row. + if (rawEvent.Channel == AuditChannel.DbOutbound + && TryGetSqlParamRedactor(opts, rawEvent.Target, out var sqlParamRegex)) + { + request = RedactSqlParameters(request, sqlParamRegex!); + } + + // --- Truncation stage ----------------------------------------- + var truncated = false; + request = TruncateField(request, cap, ref truncated); + response = TruncateField(response, cap, ref truncated); + errorDetail = TruncateField(errorDetail, cap, ref truncated); + extra = TruncateField(extra, cap, ref truncated); + + return rawEvent with + { + RequestSummary = request, + ResponseSummary = response, + ErrorDetail = errorDetail, + Extra = extra, + PayloadTruncated = rawEvent.PayloadTruncated || truncated, + }; + } + catch (Exception ex) + { + // Audit is best-effort: over-redact rather than fail the caller. + // The per-stage try/catches above already handle redactor faults + // and increment the counter; this catch covers any unexpected + // surprise in the surrounding orchestration code. + _logger.LogWarning( + ex, + "Payload filter failed; returning raw event with PayloadTruncated=true"); + try { _failureCounter.Increment(); } catch { /* swallow per §7 */ } + return rawEvent with { PayloadTruncated = true }; + } + } + + /// + /// Parse as the documented + /// {"headers": {...}, "body": ...} shape and replace values whose + /// header NAME (case-insensitive) is in with + /// . Re-serialises and returns the result. + /// + /// + /// No-op pass-through for inputs that aren't JSON-shaped — emitters that + /// have not yet adopted the convention (the M2 site emitters today, which + /// leave RequestSummary null on outbound API calls) get a transparent + /// pass. If the redactor itself throws, we over-redact the whole field + /// with and bump the failure counter. + /// + private string? RedactHeaders(string? json, IList redactList) + { + if (json is null) + { + return null; + } + + // Cheap structural pre-check: only attempt JSON parsing when the input + // actually looks like a JSON object. Saves the JsonDocument allocation + // on the (very common) non-JSON ErrorDetail / Extra fields. + var trimmed = json.AsSpan().TrimStart(); + if (trimmed.Length == 0 || trimmed[0] != '{') + { + return json; + } + + try + { + JsonNode? root; + try + { + root = JsonNode.Parse(json); + } + catch (JsonException) + { + // Not parseable JSON — leave the field alone (no error, no + // redaction). Emitters not yet using the documented shape get + // a transparent pass; Bundle C will update them. + return json; + } + + if (root is not JsonObject obj || obj["headers"] is not JsonObject headers) + { + // No "headers" object at the top level — nothing to redact. + return json; + } + + // Build a case-insensitive lookup of the redact list so we can do + // one O(1) check per header name without an inner Any() loop. + var redactSet = new HashSet(redactList, StringComparer.OrdinalIgnoreCase); + + // Take a snapshot of names first — we cannot mutate while + // enumerating the JsonObject. + var names = new List(headers.Count); + foreach (var kvp in headers) + { + names.Add(kvp.Key); + } + foreach (var name in names) + { + if (redactSet.Contains(name)) + { + headers[name] = JsonValue.Create(RedactedMarker); + } + } + + return obj.ToJsonString(RedactedSummaryJsonOptions); + } + catch (Exception ex) + { + _logger.LogWarning( + ex, + "Header redactor faulted; over-redacting field with '{Marker}'", + RedactorErrorMarker); + try { _failureCounter.Increment(); } catch { /* swallow per §7 */ } + return RedactorErrorMarker; + } + } + + /// + /// Combine the global and per-target body-redactor lists for a single + /// event, returning the compiled-regex set to apply. Patterns that failed + /// compilation are silently skipped — the compile-time failure was logged + /// once on first encounter; we never let one bad pattern starve the rest. + /// + private IReadOnlyList ResolveBodyRegexes(AuditLogOptions opts, string? target) + { + var hasGlobal = opts.GlobalBodyRedactors is { Count: > 0 }; + var perTargetAdditions = (target != null + && opts.PerTargetOverrides.TryGetValue(target, out var over) + && over.AdditionalBodyRedactors is { Count: > 0 }) + ? over.AdditionalBodyRedactors + : null; + + if (!hasGlobal && perTargetAdditions == null) + { + return Array.Empty(); + } + + var result = new List(); + if (hasGlobal) + { + foreach (var pattern in opts.GlobalBodyRedactors) + { + if (TryGetCompiledRegex(pattern, out var rx)) + { + result.Add(rx!); + } + } + } + if (perTargetAdditions != null) + { + foreach (var pattern in perTargetAdditions) + { + if (TryGetCompiledRegex(pattern, out var rx)) + { + result.Add(rx!); + } + } + } + return result; + } + + /// + /// Resolve a compiled regex from the cache, compiling it on first use. + /// Returns false for patterns that are invalid OR whose compile + /// took longer than 100 ms (the spec calls catastrophic-backtracking + /// guesses at compile time "invalid"); the failure is logged once and + /// the sentinel cache entry prevents repeat compile attempts. + /// + private bool TryGetCompiledRegex(string pattern, out Regex? regex) + { + var entry = _regexCache.GetOrAdd(pattern, CompileRegex); + regex = entry.Regex; + return entry.Regex != null; + } + + private CompiledRegex CompileRegex(string pattern) + { + try + { + var swStart = System.Diagnostics.Stopwatch.GetTimestamp(); + var rx = new Regex(pattern, RegexOptions.Compiled, RegexMatchTimeout); + var elapsedMs = (System.Diagnostics.Stopwatch.GetTimestamp() - swStart) + * 1000d / System.Diagnostics.Stopwatch.Frequency; + if (elapsedMs > 100) + { + _logger.LogWarning( + "Body redactor pattern compiled in {Elapsed}ms (> 100ms cap); rejecting '{Pattern}'", + elapsedMs, pattern); + return CompiledRegex.Invalid; + } + return new CompiledRegex(rx); + } + catch (Exception ex) + { + _logger.LogWarning( + ex, + "Body redactor pattern '{Pattern}' failed to compile; skipping", + pattern); + return CompiledRegex.Invalid; + } + } + + /// + /// Apply each compiled body-redactor regex to in + /// turn, replacing every match with . If any + /// single regex match throws (most commonly + /// ) the field is over-redacted + /// with and the failure counter is + /// incremented — the user-facing action is never aborted. + /// + private string? RedactBody(string? value, IReadOnlyList regexes) + { + if (value is null) + { + return null; + } + var current = value; + foreach (var rx in regexes) + { + try + { + current = rx.Replace(current, RedactedMarker); + } + catch (Exception ex) + { + _logger.LogWarning( + ex, + "Body redactor '{Pattern}' faulted; over-redacting field with '{Marker}'", + rx.ToString(), RedactorErrorMarker); + try { _failureCounter.Increment(); } catch { /* swallow per §7 */ } + return RedactorErrorMarker; + } + } + return current; + } + + /// + /// Resolve the per-connection SQL parameter redaction regex for the given + /// DbOutbound event target. Target shape (M4 AuditingDbCommand): the + /// connection name optionally followed by .<sql-snippet> for + /// disambiguation; the per-target dictionary is keyed by the connection + /// name alone, so we strip the snippet suffix before lookup. Patterns are + /// compiled with case-insensitive matching to match the documented + /// behaviour. + /// + private bool TryGetSqlParamRedactor(AuditLogOptions opts, string? target, out Regex? regex) + { + regex = null; + if (string.IsNullOrEmpty(target)) + { + return false; + } + + var dot = target.IndexOf('.'); + var connectionKey = dot < 0 ? target : target[..dot]; + + if (!opts.PerTargetOverrides.TryGetValue(connectionKey, out var over) + || string.IsNullOrEmpty(over.RedactSqlParamsMatching)) + { + return false; + } + + // Force case-insensitivity per the spec — even if the operator wrote + // the pattern without an IgnoreCase flag. The compile cache key folds + // the option to keep the entries unambiguous. + var cacheKey = "(?i)" + over.RedactSqlParamsMatching; + if (!TryGetCompiledRegex(cacheKey, out regex)) + { + return false; + } + return true; + } + + /// + /// Walk the M4 {"sql":"...","parameters":{...}} RequestSummary + /// shape; for each parameter whose NAME matches + /// , replace its value with + /// . Re-serialise. + /// + /// + /// No-op pass-through when the input isn't parseable JSON, isn't a JSON + /// object, or doesn't carry a top-level "parameters" object. On + /// any unexpected fault the field is over-redacted with + /// and the failure counter is bumped. + /// + private string? RedactSqlParameters(string? json, Regex paramNameRegex) + { + if (json is null) + { + return null; + } + + var trimmed = json.AsSpan().TrimStart(); + if (trimmed.Length == 0 || trimmed[0] != '{') + { + return json; + } + + try + { + JsonNode? root; + try + { + root = JsonNode.Parse(json); + } + catch (JsonException) + { + return json; + } + + if (root is not JsonObject obj || obj["parameters"] is not JsonObject parameters) + { + return json; + } + + // Snapshot the names — mutating during enumeration is unsupported. + var names = new List(parameters.Count); + foreach (var kvp in parameters) + { + names.Add(kvp.Key); + } + var anyChanged = false; + foreach (var name in names) + { + bool matched; + try + { + matched = paramNameRegex.IsMatch(name); + } + catch (Exception ex) + { + _logger.LogWarning( + ex, + "SQL parameter redactor faulted; over-redacting field with '{Marker}'", + RedactorErrorMarker); + try { _failureCounter.Increment(); } catch { /* swallow per §7 */ } + return RedactorErrorMarker; + } + if (matched) + { + parameters[name] = JsonValue.Create(RedactedMarker); + anyChanged = true; + } + } + + // Avoid re-serialising (which would normalise whitespace / order) + // when no parameter matched — keeps the on-disk row byte-identical + // to the emitter's output on the no-match path. + return anyChanged ? obj.ToJsonString(RedactedSummaryJsonOptions) : json; + } + catch (Exception ex) + { + _logger.LogWarning( + ex, + "SQL parameter redactor faulted; over-redacting field with '{Marker}'", + RedactorErrorMarker); + try { _failureCounter.Increment(); } catch { /* swallow per §7 */ } + return RedactorErrorMarker; + } + } + + private static string? TruncateField(string? value, int cap, ref bool truncated) + { + if (value is null) + { + return null; + } + var result = TruncateUtf8(value, cap); + if (result.Length != value.Length) + { + truncated = true; + } + return result; + } + + /// + /// UTF-8 byte-safe truncation. Encodes the input to UTF-8, walks back from + /// the cap position until the byte is NOT a continuation byte + /// (byte & 0xC0 == 0x80), and decodes the resulting prefix — + /// guaranteeing the returned string never splits a multi-byte sequence. + /// + private static string TruncateUtf8(string value, int capBytes) + { + if (string.IsNullOrEmpty(value)) + { + return value; + } + var bytes = Encoding.UTF8.GetBytes(value); + if (bytes.Length <= capBytes) + { + return value; + } + var boundary = capBytes; + while (boundary > 0 && (bytes[boundary] & 0xC0) == 0x80) + { + boundary--; + } + return Encoding.UTF8.GetString(bytes, 0, boundary); + } + + private static bool IsErrorStatus(AuditStatus status) => status switch + { + AuditStatus.Delivered or AuditStatus.Submitted or AuditStatus.Forwarded => false, + _ => true, + }; + + /// + /// Cache entry for a body-redactor pattern. Carries the working + /// on the success path, or the + /// sentinel for patterns that failed to compile + /// (or exceeded the 100 ms compile budget). The sentinel lets us skip + /// repeat compile attempts on every event without re-throwing on the + /// hot-path. + /// + private readonly struct CompiledRegex + { + public static readonly CompiledRegex Invalid = new(null); + + public Regex? Regex { get; } + + public CompiledRegex(Regex? regex) => Regex = regex; + } +} diff --git a/src/ScadaLink.AuditLog/Payload/IAuditPayloadFilter.cs b/src/ScadaLink.AuditLog/Payload/IAuditPayloadFilter.cs new file mode 100644 index 0000000..45b7ee2 --- /dev/null +++ b/src/ScadaLink.AuditLog/Payload/IAuditPayloadFilter.cs @@ -0,0 +1,30 @@ +using ScadaLink.Commons.Entities.Audit; + +namespace ScadaLink.AuditLog.Payload; + +/// +/// Filters an between construction and persistence — +/// truncates oversized payload fields, applies header/body/SQL-parameter +/// redaction, sets . +/// +/// +/// +/// Pure function: returns a filtered COPY of the input via with +/// expressions; never throws (over-redacts on internal failure and increments +/// the AuditRedactionFailure health metric). +/// +/// +/// Wired in M5 between event construction and the writer chain +/// (FallbackAuditWriter.WriteAsync, CentralAuditWriter.WriteAsync, +/// and the AuditLogIngestActor handlers). +/// +/// +public interface IAuditPayloadFilter +{ + /// + /// Apply the configured truncation + redaction policy to + /// and return a filtered copy. MUST NOT throw — on internal failure, over-redact + /// and surface the failure via the audit-redaction-failure health metric. + /// + AuditEvent Apply(AuditEvent rawEvent); +} diff --git a/src/ScadaLink.AuditLog/Payload/IAuditRedactionFailureCounter.cs b/src/ScadaLink.AuditLog/Payload/IAuditRedactionFailureCounter.cs new file mode 100644 index 0000000..42543ef --- /dev/null +++ b/src/ScadaLink.AuditLog/Payload/IAuditRedactionFailureCounter.cs @@ -0,0 +1,20 @@ +namespace ScadaLink.AuditLog.Payload; + +/// +/// Counter sink invoked by every time +/// a redactor (header / body regex / SQL parameter) throws and the filter has +/// to over-redact the offending field with the +/// <redacted: redactor error> marker. Bundle C bridges this into +/// the Site Health Monitoring report payload as AuditRedactionFailure. +/// +/// +/// Redaction failures must NEVER abort the user-facing action (alog.md §7) — +/// the filter over-redacts the field and surfaces the failure via this counter +/// instead. A NoOp default is the correct safe fallback while the health +/// metric is being wired in. +/// +public interface IAuditRedactionFailureCounter +{ + /// Increment the audit-redaction failure counter by one. + void Increment(); +} diff --git a/src/ScadaLink.AuditLog/Payload/NoOpAuditRedactionFailureCounter.cs b/src/ScadaLink.AuditLog/Payload/NoOpAuditRedactionFailureCounter.cs new file mode 100644 index 0000000..affeaab --- /dev/null +++ b/src/ScadaLink.AuditLog/Payload/NoOpAuditRedactionFailureCounter.cs @@ -0,0 +1,17 @@ +namespace ScadaLink.AuditLog.Payload; + +/// +/// Default binding used when the +/// Site Health Monitoring bridge has not been wired yet. Bundle C replaces +/// this registration with the real counter that surfaces in the site health +/// report payload as AuditRedactionFailure. +/// +public sealed class NoOpAuditRedactionFailureCounter : IAuditRedactionFailureCounter +{ + /// + public void Increment() + { + // Intentionally empty — Bundle C overrides this binding with the real + // health-metric counter. + } +} diff --git a/src/ScadaLink.AuditLog/ServiceCollectionExtensions.cs b/src/ScadaLink.AuditLog/ServiceCollectionExtensions.cs index 346ea0f..cf04abd 100644 --- a/src/ScadaLink.AuditLog/ServiceCollectionExtensions.cs +++ b/src/ScadaLink.AuditLog/ServiceCollectionExtensions.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using ScadaLink.AuditLog.Central; using ScadaLink.AuditLog.Configuration; +using ScadaLink.AuditLog.Payload; using ScadaLink.AuditLog.Site; using ScadaLink.AuditLog.Site.Telemetry; using ScadaLink.Commons.Interfaces.Services; @@ -59,6 +60,21 @@ public static class ServiceCollectionExtensions .ValidateOnStart(); services.AddSingleton, AuditLogOptionsValidator>(); + // M5 Bundle A: payload filter — truncates oversized RequestSummary / + // ResponseSummary / ErrorDetail / Extra fields between event + // construction and persistence. Bundle B layers header / body / + // SQL-parameter redaction onto the same singleton; Bundle C wires it + // into the FallbackAuditWriter / CentralAuditWriter / IngestActor + // paths. Singleton — the filter is stateless and the IOptionsMonitor + // dependency picks up M5-T8 hot reloads on its own. + services.AddSingleton(); + + // M5 Bundle B: per-stage redactor-failure counter. NoOp default; + // Bundle C replaces this binding with the Site Health Monitoring + // bridge that surfaces failures as AuditRedactionFailure on the site + // health report. + services.TryAddSingleton(); + // M2 Bundle E: site writer + telemetry options bindings. // BindConfiguration is not used because the configuration root supplied // by the caller may not be the application root — we go through the @@ -90,11 +106,16 @@ public static class ServiceCollectionExtensions // The script-thread surface is FallbackAuditWriter (primary + ring + // counter), not the raw SqliteAuditWriter — primary failures must NEVER // abort the user-facing action. + // Bundle C (M5-T6): the IAuditPayloadFilter singleton above is wired + // through the factory so every event written through this surface is + // truncated + redacted before it hits SQLite (and the ring on + // failure). services.AddSingleton(sp => new FallbackAuditWriter( primary: sp.GetRequiredService(), ring: sp.GetRequiredService(), failureCounter: sp.GetRequiredService(), - logger: sp.GetRequiredService>())); + logger: sp.GetRequiredService>(), + filter: sp.GetRequiredService())); // ISiteStreamAuditClient: NoOp default. M6's reconciliation work brings // the real gRPC-backed implementation (no site→central gRPC channel @@ -139,32 +160,50 @@ public static class ServiceCollectionExtensions // is intentionally distinct from IAuditWriter so site composition roots // do not accidentally bind it; central composition roots that include // AddConfigurationDatabase get a working implementation transparently. - services.AddSingleton(); + // Bundle C (M5-T6): wire the IAuditPayloadFilter into the factory so + // NotificationOutboxActor + Inbound API rows are truncated + redacted + // before they hit MS SQL. + services.AddSingleton(sp => new CentralAuditWriter( + sp, + sp.GetRequiredService>(), + sp.GetRequiredService())); return services; } /// - /// Audit Log (#23) M2 Bundle G — swap the default - /// registration for the real - /// bridge so the - /// FallbackAuditWriter primary-failure counter surfaces in the site health - /// report payload as SiteHealthReport.SiteAuditWriteFailures. + /// Audit Log (#23) M2 Bundle G + M5 Bundle C — swap the default + /// and + /// registrations for the + /// real / + /// bridges so the + /// FallbackAuditWriter primary-failure counter AND the + /// DefaultAuditPayloadFilter redactor-failure counter both surface in the + /// site health report payload as + /// SiteHealthReport.SiteAuditWriteFailures + + /// SiteHealthReport.AuditRedactionFailure. /// /// /// /// Must be called AFTER both (registers the - /// NoOp default this method replaces) and + /// NoOp defaults this method replaces) and /// ScadaLink.HealthMonitoring.ServiceCollectionExtensions.AddHealthMonitoring /// or AddSiteHealthMonitoring (registers the - /// the bridge depends on). Resolving - /// without the latter throws + /// the bridges depend on). Resolving + /// or + /// without the latter throws /// at GetRequiredService /// time — by design, since a silent NoOp would mask a misconfiguration. /// /// - /// Idempotent — calling twice replaces the descriptor each time without - /// piling up registrations. + /// Idempotent — calling twice replaces each descriptor without piling up + /// registrations. + /// + /// + /// Site-side only for M5: the central composition root keeps the NoOp + /// defaults; the central health-metric surface that would expose + /// AuditRedactionFailure next to the existing central counters + /// ships in M6. /// /// public static IServiceCollection AddAuditLogHealthMetricsBridge(this IServiceCollection services) @@ -173,6 +212,8 @@ public static class ServiceCollectionExtensions services.Replace( ServiceDescriptor.Singleton()); + services.Replace( + ServiceDescriptor.Singleton()); return services; } } diff --git a/src/ScadaLink.AuditLog/Site/FallbackAuditWriter.cs b/src/ScadaLink.AuditLog/Site/FallbackAuditWriter.cs index 9b911c5..18511f1 100644 --- a/src/ScadaLink.AuditLog/Site/FallbackAuditWriter.cs +++ b/src/ScadaLink.AuditLog/Site/FallbackAuditWriter.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.Logging; +using ScadaLink.AuditLog.Payload; using ScadaLink.Commons.Entities.Audit; using ScadaLink.Commons.Interfaces.Services; @@ -30,27 +31,48 @@ public sealed class FallbackAuditWriter : IAuditWriter private readonly RingBufferFallback _ring; private readonly IAuditWriteFailureCounter _failureCounter; private readonly ILogger _logger; + private readonly IAuditPayloadFilter? _filter; private readonly SemaphoreSlim _drainGate = new(1, 1); + /// + /// Bundle C (M5-T6) wires the singleton + /// here so every event written via the site hot path is truncated + + /// header/body/SQL-param redacted before it hits both the primary SQLite + /// writer AND the ring fallback. The parameter is optional (defaults to + /// no filtering) so the long tail of test composition roots that don't + /// care about the filter need no change — the production + /// registration + /// always passes the real filter through. + /// public FallbackAuditWriter( IAuditWriter primary, RingBufferFallback ring, IAuditWriteFailureCounter failureCounter, - ILogger logger) + ILogger logger, + IAuditPayloadFilter? filter = null) { _primary = primary ?? throw new ArgumentNullException(nameof(primary)); _ring = ring ?? throw new ArgumentNullException(nameof(ring)); _failureCounter = failureCounter ?? throw new ArgumentNullException(nameof(failureCounter)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _filter = filter; // null = no-op pass-through; see WriteAsync. } public async Task WriteAsync(AuditEvent evt, CancellationToken ct = default) { ArgumentNullException.ThrowIfNull(evt); + // Filter once, up-front. The filtered event flows BOTH to the primary + // and (on failure) to the ring buffer — so a primary outage that + // drains later still hands the SqliteAuditWriter a row that has + // already been truncated and redacted. The filter contract is + // "MUST NOT throw"; the null-coalesce keeps test composition roots + // that don't wire a filter working unchanged. + var filtered = _filter?.Apply(evt) ?? evt; + try { - await _primary.WriteAsync(evt, ct).ConfigureAwait(false); + await _primary.WriteAsync(filtered, ct).ConfigureAwait(false); } catch (Exception ex) { @@ -62,8 +84,12 @@ public sealed class FallbackAuditWriter : IAuditWriter _failureCounter.Increment(); _logger.LogWarning(ex, "Primary audit writer threw; routing EventId {EventId} to drop-oldest ring.", - evt.EventId); - _ring.TryEnqueue(evt); + filtered.EventId); + // Ring stores the filtered copy so the eventual drain replays a + // payload that has already been capped/redacted — no second + // filter pass needed on recovery, and no risk of the ring + // holding the raw oversized blob in memory. + _ring.TryEnqueue(filtered); return; } diff --git a/src/ScadaLink.AuditLog/Site/HealthMetricsAuditRedactionFailureCounter.cs b/src/ScadaLink.AuditLog/Site/HealthMetricsAuditRedactionFailureCounter.cs new file mode 100644 index 0000000..78454e3 --- /dev/null +++ b/src/ScadaLink.AuditLog/Site/HealthMetricsAuditRedactionFailureCounter.cs @@ -0,0 +1,48 @@ +using ScadaLink.AuditLog.Payload; +using ScadaLink.HealthMonitoring; + +namespace ScadaLink.AuditLog.Site; + +/// +/// Audit Log (#23) M5 Bundle C — bridges +/// (incremented by +/// every time a header / body / SQL +/// parameter redactor stage throws and the filter has to over-redact the +/// offending field) into so the count +/// surfaces in the site health report payload as +/// SiteHealthReport.AuditRedactionFailure. +/// +/// +/// +/// Registered by ; +/// callers must register AddHealthMonitoring() first so +/// resolves. The default +/// registration keeps for nodes +/// where Site Health Monitoring is not wired (the silent-sink contract — +/// redaction failures must NEVER abort the user-facing action, alog.md §7). +/// +/// +/// Mirrors the M2 Bundle G +/// shape one-for-one so the two health-metric bridges age together. +/// +/// +/// Site-side only for M5: the redaction filter also runs on the central +/// writers (CentralAuditWriter + AuditLogIngestActor), but the central +/// health-metric surface that would expose AuditRedactionFailure +/// alongside the existing central counters ships in M6. Until then, the +/// central composition root keeps the NoOp default — the redactions still +/// happen, they just don't get counted into a health report. +/// +/// +public sealed class HealthMetricsAuditRedactionFailureCounter : IAuditRedactionFailureCounter +{ + private readonly ISiteHealthCollector _collector; + + public HealthMetricsAuditRedactionFailureCounter(ISiteHealthCollector collector) + { + _collector = collector ?? throw new ArgumentNullException(nameof(collector)); + } + + /// + public void Increment() => _collector.IncrementAuditRedactionFailure(); +} diff --git a/src/ScadaLink.Commons/Messages/Health/SiteHealthReport.cs b/src/ScadaLink.Commons/Messages/Health/SiteHealthReport.cs index 516d4f3..bba4c8d 100644 --- a/src/ScadaLink.Commons/Messages/Health/SiteHealthReport.cs +++ b/src/ScadaLink.Commons/Messages/Health/SiteHealthReport.cs @@ -25,7 +25,14 @@ public record SiteHealthReport( // primary failures (SQLite throws routed to the drop-oldest ring). Surfaces // a sustained audit-write outage on /monitoring/health. Defaults to 0 so // existing producers / tests that don't construct the field stay valid. - int SiteAuditWriteFailures = 0); + int SiteAuditWriteFailures = 0, + // Audit Log (#23) M5 Bundle C: per-interval count of payload-filter + // redactor over-redactions (header / body / SQL parameter stages all + // throwing → field replaced with the "" + // marker). Surfaces a misconfigured / catastrophic regex on + // /monitoring/health. Defaults to 0 for back-compat with existing + // producers and tests that don't construct the field. + int AuditRedactionFailure = 0); /// /// Broadcast wrapper used between central nodes to keep per-node diff --git a/src/ScadaLink.HealthMonitoring/ISiteHealthCollector.cs b/src/ScadaLink.HealthMonitoring/ISiteHealthCollector.cs index c16c45f..bcd5f9e 100644 --- a/src/ScadaLink.HealthMonitoring/ISiteHealthCollector.cs +++ b/src/ScadaLink.HealthMonitoring/ISiteHealthCollector.cs @@ -19,6 +19,15 @@ public interface ISiteHealthCollector /// AddAuditLogHealthMetricsBridge(). /// void IncrementSiteAuditWriteFailures(); + /// + /// Audit Log (#23) M5 Bundle C — increment the per-interval count of + /// payload-filter redactor over-redactions (header / body / SQL + /// parameter stage throws routed to the + /// <redacted: redactor error> marker). Bridged from the + /// IAuditRedactionFailureCounter binding registered via + /// AddAuditLogHealthMetricsBridge(). + /// + void IncrementAuditRedactionFailure(); void UpdateConnectionHealth(string connectionName, ConnectionHealth health); void RemoveConnection(string connectionName); void UpdateTagResolution(string connectionName, int totalSubscribed, int successfullyResolved); diff --git a/src/ScadaLink.HealthMonitoring/SiteHealthCollector.cs b/src/ScadaLink.HealthMonitoring/SiteHealthCollector.cs index 1a6aa48..47567c9 100644 --- a/src/ScadaLink.HealthMonitoring/SiteHealthCollector.cs +++ b/src/ScadaLink.HealthMonitoring/SiteHealthCollector.cs @@ -14,6 +14,7 @@ public class SiteHealthCollector : ISiteHealthCollector private int _alarmErrorCount; private int _deadLetterCount; private int _siteAuditWriteFailures; + private int _auditRedactionFailures; private readonly ConcurrentDictionary _connectionStatuses = new(); private readonly ConcurrentDictionary _tagResolutionCounts = new(); private readonly ConcurrentDictionary _connectionEndpoints = new(); @@ -74,6 +75,20 @@ public class SiteHealthCollector : ISiteHealthCollector Interlocked.Increment(ref _siteAuditWriteFailures); } + /// + /// Audit Log (#23) M5 Bundle C — increment the per-interval count of + /// payload-filter redactor over-redactions (header / body / SQL + /// parameter stages routed to the + /// <redacted: redactor error> marker). Bridged from the + /// IAuditRedactionFailureCounter binding registered via + /// AddAuditLogHealthMetricsBridge(); reset every interval together + /// with the other per-interval counters. + /// + public void IncrementAuditRedactionFailure() + { + Interlocked.Increment(ref _auditRedactionFailures); + } + /// /// Update the health status for a named data connection. /// Called by DCL when connection state changes. @@ -158,6 +173,7 @@ public class SiteHealthCollector : ISiteHealthCollector var alarmErrors = Interlocked.Exchange(ref _alarmErrorCount, 0); var deadLetters = Interlocked.Exchange(ref _deadLetterCount, 0); var siteAuditWriteFailures = Interlocked.Exchange(ref _siteAuditWriteFailures, 0); + var auditRedactionFailures = Interlocked.Exchange(ref _auditRedactionFailures, 0); // Snapshot current connection and tag resolution state var connectionStatuses = new Dictionary(_connectionStatuses); @@ -190,6 +206,7 @@ public class SiteHealthCollector : ISiteHealthCollector DataConnectionTagQuality: tagQuality, ParkedMessageCount: Interlocked.CompareExchange(ref _parkedMessageCount, 0, 0), ClusterNodes: _clusterNodes?.ToList(), - SiteAuditWriteFailures: siteAuditWriteFailures); + SiteAuditWriteFailures: siteAuditWriteFailures, + AuditRedactionFailure: auditRedactionFailures); } } diff --git a/tests/ScadaLink.AuditLog.Tests/Configuration/AuditLogOptionsBindingTests.cs b/tests/ScadaLink.AuditLog.Tests/Configuration/AuditLogOptionsBindingTests.cs new file mode 100644 index 0000000..f9829cd --- /dev/null +++ b/tests/ScadaLink.AuditLog.Tests/Configuration/AuditLogOptionsBindingTests.cs @@ -0,0 +1,220 @@ +using System.Text; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using ScadaLink.AuditLog.Configuration; +using ScadaLink.AuditLog.Payload; +using ScadaLink.Commons.Entities.Audit; +using ScadaLink.Commons.Types.Enums; + +namespace ScadaLink.AuditLog.Tests.Configuration; + +/// +/// Bundle D (M5-T8) tests for hot-reloadable +/// binding. The first test pins the JSON-realistic binding shape end-to-end +/// (scalars, lists, per-target overrides) so accidental drift in the section +/// layout breaks the build. The second test exercises the live hot-reload +/// path: a backed by a mutable +/// must respond to config changes on +/// the very next event, with both cap-bytes and the regex-cache invalidation +/// flowing through without a restart. +/// +/// +/// Distinct from (M1-T9) which covered +/// section binding + validator failures via single-key in-memory config — those +/// tests exist; these add (a) end-to-end binding from a realistic JSON literal +/// and (b) the hot-reload behavioural contract the M5-T8 spec calls out. +/// +public class AuditLogOptionsBindingTests +{ + [Fact] + public void AuditLog_Section_Binds_AllFields() + { + const string json = """ + { + "AuditLog": { + "DefaultCapBytes": 4096, + "ErrorCapBytes": 32768, + "HeaderRedactList": ["Authorization", "Custom-Token"], + "GlobalBodyRedactors": ["\"password\":\\s*\"[^\"]*\""], + "PerTargetOverrides": { + "myconnection": { + "CapBytes": 16384, + "AdditionalBodyRedactors": [], + "RedactSqlParamsMatching": "@token|@secret" + } + }, + "RetentionDays": 180 + } + } + """; + + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json)); + var configuration = new ConfigurationBuilder() + .AddJsonStream(stream) + .Build(); + var services = new ServiceCollection(); + services.AddAuditLog(configuration); + using var provider = services.BuildServiceProvider(); + + var opts = provider.GetRequiredService>().Value; + + // Scalars. + Assert.Equal(4096, opts.DefaultCapBytes); + Assert.Equal(32768, opts.ErrorCapBytes); + Assert.Equal(180, opts.RetentionDays); + + // HeaderRedactList: the Microsoft.Extensions.Configuration list binder + // APPENDS to the default list, so we assert containment rather than + // exact equality (see M1-T9 AuditLogOptionsTests for the rationale). + Assert.Contains("Authorization", opts.HeaderRedactList); + Assert.Contains("Custom-Token", opts.HeaderRedactList); + + // GlobalBodyRedactors: pattern arrived intact, regex-escape sequences + // and all. + Assert.Contains("\"password\":\\s*\"[^\"]*\"", opts.GlobalBodyRedactors); + + // PerTargetOverrides: keyed by connection name, each field bound. + Assert.True(opts.PerTargetOverrides.ContainsKey("myconnection")); + var ov = opts.PerTargetOverrides["myconnection"]; + Assert.Equal(16384, ov.CapBytes); + // Microsoft.Extensions.Configuration JSON binder leaves an empty array + // null on a nullable List; either null or empty is acceptable as + // "no additional redactors" — both result in zero patterns at use. + Assert.True(ov.AdditionalBodyRedactors is null || ov.AdditionalBodyRedactors.Count == 0); + Assert.Equal("@token|@secret", ov.RedactSqlParamsMatching); + } + + [Fact] + public void Filter_Behavior_Updates_OnConfigReload() + { + // Start at the default cap (4096). A 5 KB body should be truncated; + // PayloadTruncated flips to true. + var initial = new AuditLogOptions { DefaultCapBytes = 4096 }; + var monitor = new TestOptionsMonitor(initial); + var filter = new DefaultAuditPayloadFilter( + monitor, + NullLogger.Instance); + + var body = new string('x', 5 * 1024); + var evt = new AuditEvent + { + EventId = Guid.NewGuid(), + OccurredAtUtc = DateTime.UtcNow, + Channel = AuditChannel.ApiOutbound, + Kind = AuditKind.ApiCall, + Status = AuditStatus.Delivered, + RequestSummary = body, + }; + + var resultBefore = filter.Apply(evt); + Assert.True(resultBefore.PayloadTruncated, "5KB body at 4096 cap must be truncated"); + Assert.NotNull(resultBefore.RequestSummary); + Assert.True(Encoding.UTF8.GetByteCount(resultBefore.RequestSummary!) <= 4096); + + // Reload: cap raised to 16384 — next event must NOT truncate. This is + // the M5-T8 contract: the filter sees the new value on the very next + // Apply, without process restart. + monitor.Set(new AuditLogOptions { DefaultCapBytes = 16384 }); + + var resultAfter = filter.Apply(evt); + Assert.False(resultAfter.PayloadTruncated, "5KB body at 16384 cap must NOT be truncated"); + Assert.Equal(body, resultAfter.RequestSummary); + } + + [Fact] + public void Filter_PicksUp_NewBodyRedactor_OnConfigReload() + { + // The regex cache is keyed by pattern string — a redactor added via + // config reload must compile + apply on the very next event without a + // process restart. Pre-reload: no redactor, hunter2 survives. After + // reload: hunter2 redacted. + var monitor = new TestOptionsMonitor(new AuditLogOptions()); + var filter = new DefaultAuditPayloadFilter( + monitor, + NullLogger.Instance); + + const string body = "{\"user\":\"alice\",\"password\":\"hunter2\"}"; + var evt = new AuditEvent + { + EventId = Guid.NewGuid(), + OccurredAtUtc = DateTime.UtcNow, + Channel = AuditChannel.ApiOutbound, + Kind = AuditKind.ApiCall, + Status = AuditStatus.Delivered, + RequestSummary = body, + }; + + var before = filter.Apply(evt); + Assert.Contains("hunter2", before.RequestSummary!); + + monitor.Set(new AuditLogOptions + { + GlobalBodyRedactors = new List { "\"password\":\\s*\"[^\"]*\"" }, + }); + + var after = filter.Apply(evt); + Assert.DoesNotContain("hunter2", after.RequestSummary!); + Assert.Contains("", after.RequestSummary!); + } + + /// + /// IOptionsMonitor test double — exposes a method that + /// updates the current value and fires registered OnChange callbacks. + /// Avoids depending on Microsoft.Extensions.Configuration's reload-token + /// plumbing, which is awkward to drive deterministically from xUnit. + /// + private sealed class TestOptionsMonitor : IOptionsMonitor + { + private T _current; + private readonly List> _listeners = new(); + + public TestOptionsMonitor(T initial) => _current = initial; + + public T CurrentValue => _current; + + public T Get(string? name) => _current; + + public IDisposable? OnChange(Action listener) + { + lock (_listeners) + { + _listeners.Add(listener); + } + return new Unsubscribe(_listeners, listener); + } + + public void Set(T value) + { + _current = value; + Action[] snapshot; + lock (_listeners) + { + snapshot = _listeners.ToArray(); + } + foreach (var l in snapshot) + { + l(_current, Options.DefaultName); + } + } + + private sealed class Unsubscribe : IDisposable + { + private readonly List> _listeners; + private readonly Action _listener; + public Unsubscribe(List> listeners, Action listener) + { + _listeners = listeners; + _listener = listener; + } + public void Dispose() + { + lock (_listeners) + { + _listeners.Remove(_listener); + } + } + } + } +} diff --git a/tests/ScadaLink.AuditLog.Tests/Payload/BodyRegexRedactionTests.cs b/tests/ScadaLink.AuditLog.Tests/Payload/BodyRegexRedactionTests.cs new file mode 100644 index 0000000..85393d1 --- /dev/null +++ b/tests/ScadaLink.AuditLog.Tests/Payload/BodyRegexRedactionTests.cs @@ -0,0 +1,207 @@ +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using ScadaLink.AuditLog.Configuration; +using ScadaLink.AuditLog.Payload; +using ScadaLink.Commons.Entities.Audit; +using ScadaLink.Commons.Types.Enums; + +namespace ScadaLink.AuditLog.Tests.Payload; + +/// +/// Bundle B (M5-T4) tests for body regex redaction in +/// . The body-redactor stage runs +/// regex replace against RequestSummary / ResponseSummary / ErrorDetail / +/// Extra, replacing every match with <redacted>. Regexes come +/// from plus the per-target +/// . Each +/// regex is compiled with a 50 ms timeout so catastrophic-backtracking +/// patterns trip a ; +/// when that happens the offending field is over-redacted with +/// <redacted: redactor error> and the +/// is incremented. The stage runs +/// BEFORE truncation. +/// +public class BodyRegexRedactionTests +{ + private static IOptionsMonitor Monitor(AuditLogOptions? opts = null) => + new StaticMonitor(opts ?? new AuditLogOptions()); + + private static DefaultAuditPayloadFilter Filter( + AuditLogOptions? opts = null, + IAuditRedactionFailureCounter? counter = null) => + new(Monitor(opts), NullLogger.Instance, counter); + + private static AuditEvent NewEvent( + AuditStatus status = AuditStatus.Delivered, + string? request = null, + string? response = null, + string? errorDetail = null, + string? extra = null, + string? target = null) => new() + { + EventId = Guid.NewGuid(), + OccurredAtUtc = DateTime.UtcNow, + Channel = AuditChannel.ApiOutbound, + Kind = AuditKind.ApiCall, + Status = status, + Target = target, + RequestSummary = request, + ResponseSummary = response, + ErrorDetail = errorDetail, + Extra = extra, + }; + + [Fact] + public void GlobalRegex_HunterPassword_Redacted() + { + var opts = new AuditLogOptions + { + GlobalBodyRedactors = new List { "\"password\":\\s*\"[^\"]*\"" }, + }; + const string input = "{\"user\":\"alice\",\"password\":\"hunter2\"}"; + var evt = NewEvent(request: input); + + var result = Filter(opts).Apply(evt); + + Assert.NotNull(result.RequestSummary); + Assert.Contains("", result.RequestSummary); + Assert.DoesNotContain("hunter2", result.RequestSummary); + Assert.Contains("alice", result.RequestSummary); + } + + [Fact] + public void PerTargetRegex_OnlyAppliedToMatchingTarget() + { + var opts = new AuditLogOptions + { + PerTargetOverrides = new Dictionary + { + ["esg.A"] = new PerTargetRedactionOverride + { + AdditionalBodyRedactors = new List { "SECRET-[A-Z0-9]+" }, + }, + }, + }; + + const string input = "token=SECRET-XYZ123 normal-text"; + + var matchedEvt = NewEvent(request: input, target: "esg.A"); + var matchedResult = Filter(opts).Apply(matchedEvt); + Assert.Contains("", matchedResult.RequestSummary!); + Assert.DoesNotContain("SECRET-XYZ123", matchedResult.RequestSummary!); + + var unmatchedEvt = NewEvent(request: input, target: "esg.B"); + var unmatchedResult = Filter(opts).Apply(unmatchedEvt); + Assert.Equal(input, unmatchedResult.RequestSummary); + } + + [Fact] + public void RegexThrowsTimeout_FieldBecomesRedactedMarker_CounterIncrements() + { + // Catastrophic backtracking pattern: alternation with overlapping + // groups + non-matching suffix forces the engine into exponential + // work that blows past the 50 ms timeout. Append a non-'a' character + // so the suffix anchor fails and the engine has to exhaust every + // permutation. + var opts = new AuditLogOptions + { + GlobalBodyRedactors = new List { "^(a+)+$" }, + }; + // 30 'a's followed by '!' — small enough to keep the test fast, big + // enough to overflow the 50 ms regex timeout on every machine the CI + // grid runs on. + var input = new string('a', 30) + "!"; + var counter = new CountingRedactionFailureCounter(); + var evt = NewEvent(request: input); + + var result = Filter(opts, counter).Apply(evt); + + Assert.Equal("", result.RequestSummary); + Assert.True(counter.Count >= 1, $"expected counter >= 1, got {counter.Count}"); + } + + [Fact] + public void NoRegexConfigured_FieldUnchanged() + { + var opts = new AuditLogOptions(); // no GlobalBodyRedactors, no per-target + const string input = "{\"password\":\"hunter2\"}"; + var evt = NewEvent(request: input); + + var result = Filter(opts).Apply(evt); + + Assert.Equal(input, result.RequestSummary); + } + + [Fact] + public void RedactionAppliedBeforeTruncation() + { + // A pattern that matches a long secret in the body. The full input is + // > 8 KB so truncation must run. After redaction: + // * the marker survives the cap (redaction ran first), + // * the original secret bytes do NOT survive, + // * PayloadTruncated is set. + var opts = new AuditLogOptions + { + GlobalBodyRedactors = new List { "SECRET-[A-Z0-9]+" }, + }; + var secret = "SECRET-ABCDEF123"; + var padding = new string('x', 9 * 1024); + var input = secret + padding; + Assert.True(Encoding.UTF8.GetByteCount(input) > 8192); + + var evt = NewEvent(AuditStatus.Delivered, request: input); + + var result = Filter(opts).Apply(evt); + + Assert.NotNull(result.RequestSummary); + Assert.True(Encoding.UTF8.GetByteCount(result.RequestSummary!) <= 8192); + Assert.Contains("", result.RequestSummary); + Assert.DoesNotContain(secret, result.RequestSummary); + Assert.True(result.PayloadTruncated); + } + + [Fact] + public void CatastrophicBacktrackingRegex_AtCompileTime_RejectedAtStartup() + { + // .NET's regex engine has no compile-time detection for catastrophic + // backtracking (only structural validation), so the filter's + // protection is RUNTIME — the 50 ms per-match timeout. We assert the + // safety net behaviour: a known evil pattern compiles cleanly but + // matches time out at runtime, the field is over-redacted, and the + // failure counter is incremented. Future engines that DO support + // compile-time analysis can tighten this further; the contract here + // is that the user-facing action is never aborted. + var evilPattern = "^(a+)+$"; + var opts = new AuditLogOptions + { + GlobalBodyRedactors = new List { evilPattern }, + }; + var input = new string('a', 30) + "!"; + var counter = new CountingRedactionFailureCounter(); + var evt = NewEvent(request: input); + + var result = Filter(opts, counter).Apply(evt); + + Assert.Equal("", result.RequestSummary); + Assert.True(counter.Count >= 1); + } + + /// Test double that counts increments. + private sealed class CountingRedactionFailureCounter : IAuditRedactionFailureCounter + { + private int _count; + public int Count => _count; + public void Increment() => System.Threading.Interlocked.Increment(ref _count); + } + + /// IOptionsMonitor test double — returns the same snapshot on every read. + private sealed class StaticMonitor : IOptionsMonitor + { + private readonly AuditLogOptions _value; + public StaticMonitor(AuditLogOptions value) => _value = value; + public AuditLogOptions CurrentValue => _value; + public AuditLogOptions Get(string? name) => _value; + public IDisposable? OnChange(Action listener) => null; + } +} diff --git a/tests/ScadaLink.AuditLog.Tests/Payload/FilterIntegrationTests.cs b/tests/ScadaLink.AuditLog.Tests/Payload/FilterIntegrationTests.cs new file mode 100644 index 0000000..ca3aaab --- /dev/null +++ b/tests/ScadaLink.AuditLog.Tests/Payload/FilterIntegrationTests.cs @@ -0,0 +1,301 @@ +using System.Text; +using Akka.Actor; +using Akka.TestKit.Xunit2; +using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using NSubstitute; +using ScadaLink.AuditLog.Central; +using ScadaLink.AuditLog.Configuration; +using ScadaLink.AuditLog.Payload; +using ScadaLink.AuditLog.Site; +using ScadaLink.Commons.Entities.Audit; +using ScadaLink.Commons.Interfaces.Repositories; +using ScadaLink.Commons.Interfaces.Services; +using ScadaLink.Commons.Messages.Audit; +using ScadaLink.Commons.Types; +using ScadaLink.Commons.Types.Enums; +using ScadaLink.ConfigurationDatabase; +using ScadaLink.ConfigurationDatabase.Repositories; +using ScadaLink.ConfigurationDatabase.Tests.Migrations; + +namespace ScadaLink.AuditLog.Tests.Payload; + +/// +/// Bundle C (M5-T6) integration tests verifying that the +/// wires correctly into each of the three +/// writer entry points — on the site hot +/// path, on the central direct-write path, +/// and on the site→central telemetry ingest +/// path (both the per-row IngestAuditEventsCommand handler and the +/// combined IngestCachedTelemetryCommand dual-write handler). +/// +/// +/// Bundle B established the filter's behaviour in isolation (truncation, +/// header redaction, body-regex redaction, SQL-parameter redaction). Bundle C +/// proves that filtering actually happens before persistence — a 10 KB +/// RequestSummary on a Delivered row must land on disk capped to 8192 bytes +/// with PayloadTruncated=true, regardless of whether the row was +/// written via the site's SQLite hot path, the central direct-write path, or +/// the site→central ingest pipeline. We use the production +/// through every test so the +/// integration is real end-to-end, not a fake-filter assertion. +/// +public class FilterIntegrationTests +{ + /// + /// Default-options filter — 8 KiB cap on success rows, 64 KiB on error + /// rows. Cached and reused; the filter is stateless w.r.t. the per-event + /// inputs and the regex cache is happy under sharing. + /// + private static IAuditPayloadFilter NewDefaultFilter() + { + var monitor = Microsoft.Extensions.Options.Options.Create(new AuditLogOptions()); + return new DefaultAuditPayloadFilter( + new StaticMonitor(monitor.Value), + NullLogger.Instance); + } + + private static AuditEvent NewEvent(string? request = null, Guid? eventId = null) => new() + { + EventId = eventId ?? Guid.NewGuid(), + OccurredAtUtc = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc), + Channel = AuditChannel.ApiOutbound, + Kind = AuditKind.ApiCall, + // Delivered = success cap (8 KiB). Picking a success status so the + // 10 KB payload reliably trips the filter. + Status = AuditStatus.Delivered, + RequestSummary = request, + PayloadTruncated = false, + ForwardState = AuditForwardState.Pending, + }; + + // -- C1.1: FallbackAuditWriter applies the filter before SQLite write ---- + + [Fact] + public async Task FallbackAuditWriter_AppliesFilter_BeforeSqliteWrite() + { + var dataSource = + $"file:filter-fbw-{Guid.NewGuid():N}?mode=memory&cache=shared"; + // Hold the in-memory database alive for the verifier connection — + // SQLite frees a Cache=Shared in-memory DB when the last connection + // closes, so without this keep-alive the FallbackAuditWriter's + // dispose would wipe the data before we could query it. + using var keepAlive = new SqliteConnection($"Data Source={dataSource};Cache=Shared"); + keepAlive.Open(); + + var sqliteWriter = new SqliteAuditWriter( + Microsoft.Extensions.Options.Options.Create(new SqliteAuditWriterOptions { DatabasePath = dataSource }), + NullLogger.Instance, + connectionStringOverride: $"Data Source={dataSource};Cache=Shared"); + await using var _disposeSqlite = sqliteWriter; + + var fallback = new FallbackAuditWriter( + sqliteWriter, + new RingBufferFallback(), + new NoOpAuditWriteFailureCounter(), + NullLogger.Instance, + NewDefaultFilter()); + + var bigRequest = new string('a', 10 * 1024); + var evt = NewEvent(request: bigRequest); + await fallback.WriteAsync(evt); + + // Read back via a fresh connection so we observe what actually + // landed in SQLite — not what the writer was handed. + using var verifier = new SqliteConnection($"Data Source={dataSource};Cache=Shared"); + verifier.Open(); + using var cmd = verifier.CreateCommand(); + cmd.CommandText = "SELECT RequestSummary, PayloadTruncated FROM AuditLog WHERE EventId = $id;"; + cmd.Parameters.AddWithValue("$id", evt.EventId.ToString()); + using var reader = cmd.ExecuteReader(); + Assert.True(reader.Read()); + var persistedRequest = reader.GetString(0); + var truncatedFlag = reader.GetInt32(1); + + Assert.Equal(8192, Encoding.UTF8.GetByteCount(persistedRequest)); + Assert.Equal(1, truncatedFlag); + } + + // -- C1.2: CentralAuditWriter applies the filter before repo insert ------ + + [Fact] + public async Task CentralAuditWriter_AppliesFilter_BeforeRepoInsert() + { + var repo = Substitute.For(); + var services = new ServiceCollection(); + services.AddScoped(_ => repo); + services.AddSingleton(NewDefaultFilter()); + var provider = services.BuildServiceProvider(); + + var writer = new CentralAuditWriter( + provider, NullLogger.Instance, NewDefaultFilter()); + + var bigRequest = new string('b', 10 * 1024); + var evt = NewEvent(request: bigRequest); + await writer.WriteAsync(evt); + + // Verify the repository saw the FILTERED event, not the raw one. + // The filter caps RequestSummary to 8192 bytes on a Delivered row + // and flags PayloadTruncated. + await repo.Received(1).InsertIfNotExistsAsync( + Arg.Is(e => + e.EventId == evt.EventId + && e.RequestSummary != null + && Encoding.UTF8.GetByteCount(e.RequestSummary) == 8192 + && e.PayloadTruncated == true), + Arg.Any()); + } + + // -- C1.3 + C1.4: AuditLogIngestActor applies the filter on both paths --- + + public class IngestActorTests : TestKit, IClassFixture + { + private readonly MsSqlMigrationFixture _fixture; + + public IngestActorTests(MsSqlMigrationFixture fixture) + { + _fixture = fixture; + } + + private ScadaLinkDbContext CreateReadContext() + { + var options = new DbContextOptionsBuilder() + .UseSqlServer(_fixture.ConnectionString) + .Options; + return new ScadaLinkDbContext(options); + } + + private static string NewSiteId() => + "test-bundle-c1-filter-" + Guid.NewGuid().ToString("N").Substring(0, 8); + + /// + /// Build the IServiceProvider in the production-flavoured shape — + /// scoped repositories + a singleton + /// resolved per-message from the actor's scope. Matches the + /// AddAuditLog registrations Bundle B established. + /// + private IServiceProvider BuildServiceProvider() + { + var services = new ServiceCollection(); + services.AddDbContext(opts => + opts.UseSqlServer(_fixture.ConnectionString) + .ConfigureWarnings(w => w.Ignore( + Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning))); + services.AddScoped(sp => + new AuditLogRepository(sp.GetRequiredService())); + services.AddScoped(sp => + new SiteCallAuditRepository(sp.GetRequiredService())); + services.AddSingleton(NewDefaultFilter()); + return services.BuildServiceProvider(); + } + + [SkippableFact] + public async Task AuditLogIngestActor_AppliesFilter_BeforeBatchInsert() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + var siteId = NewSiteId(); + var bigRequest = new string('c', 10 * 1024); + var evt = new AuditEvent + { + EventId = Guid.NewGuid(), + OccurredAtUtc = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc), + Channel = AuditChannel.ApiOutbound, + Kind = AuditKind.ApiCall, + Status = AuditStatus.Delivered, + SourceSiteId = siteId, + RequestSummary = bigRequest, + PayloadTruncated = false, + }; + + var sp = BuildServiceProvider(); + var actor = Sys.ActorOf(Props.Create(() => new AuditLogIngestActor( + sp, NullLogger.Instance))); + + actor.Tell(new IngestAuditEventsCommand(new[] { evt }), TestActor); + ExpectMsg(TimeSpan.FromSeconds(15)); + + // Verify the persisted row was filtered before INSERT. + await using var read = CreateReadContext(); + var row = await read.Set() + .SingleAsync(e => e.EventId == evt.EventId); + Assert.NotNull(row.RequestSummary); + Assert.Equal(8192, Encoding.UTF8.GetByteCount(row.RequestSummary!)); + Assert.True(row.PayloadTruncated); + } + + [SkippableFact] + public async Task AuditLogIngestActor_CachedTelemetry_AppliesFilter() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + var siteId = NewSiteId(); + var trackedId = TrackedOperationId.New(); + var bigRequest = new string('d', 10 * 1024); + var audit = new AuditEvent + { + EventId = Guid.NewGuid(), + OccurredAtUtc = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc), + Channel = AuditChannel.ApiOutbound, + Kind = AuditKind.CachedSubmit, + Status = AuditStatus.Submitted, + SourceSiteId = siteId, + CorrelationId = trackedId.Value, + RequestSummary = bigRequest, + PayloadTruncated = false, + }; + var siteCall = new SiteCall + { + TrackedOperationId = trackedId, + Channel = "ApiOutbound", + Target = "ERP.GetOrder", + SourceSite = siteId, + Status = "Submitted", + RetryCount = 0, + CreatedAtUtc = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc), + UpdatedAtUtc = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc), + IngestedAtUtc = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc), + }; + + var sp = BuildServiceProvider(); + var actor = Sys.ActorOf(Props.Create(() => new AuditLogIngestActor( + sp, NullLogger.Instance))); + + actor.Tell( + new IngestCachedTelemetryCommand(new[] { new CachedTelemetryEntry(audit, siteCall) }), + TestActor); + ExpectMsg(TimeSpan.FromSeconds(15)); + + await using var read = CreateReadContext(); + var auditRow = await read.Set() + .SingleAsync(e => e.EventId == audit.EventId); + Assert.NotNull(auditRow.RequestSummary); + // Bundle C filter must run before the dual-write transaction + // commits, so the persisted AuditLog row carries the truncated + // payload. + Assert.Equal(8192, Encoding.UTF8.GetByteCount(auditRow.RequestSummary!)); + Assert.True(auditRow.PayloadTruncated); + } + } + + /// + /// IOptionsMonitor test double — returns the same snapshot on every read, + /// no change-token plumbing required for these tests. Mirrors the helper + /// used in TruncationTests. + /// + private sealed class StaticMonitor : IOptionsMonitor + { + private readonly AuditLogOptions _value; + + public StaticMonitor(AuditLogOptions value) => _value = value; + + public AuditLogOptions CurrentValue => _value; + + public AuditLogOptions Get(string? name) => _value; + + public IDisposable? OnChange(Action listener) => null; + } +} diff --git a/tests/ScadaLink.AuditLog.Tests/Payload/HeaderRedactionTests.cs b/tests/ScadaLink.AuditLog.Tests/Payload/HeaderRedactionTests.cs new file mode 100644 index 0000000..01784f6 --- /dev/null +++ b/tests/ScadaLink.AuditLog.Tests/Payload/HeaderRedactionTests.cs @@ -0,0 +1,217 @@ +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using ScadaLink.AuditLog.Configuration; +using ScadaLink.AuditLog.Payload; +using ScadaLink.Commons.Entities.Audit; +using ScadaLink.Commons.Types.Enums; + +namespace ScadaLink.AuditLog.Tests.Payload; + +/// +/// Bundle B (M5-T3) tests for HTTP header +/// redaction. Redaction parses / +/// as JSON of shape +/// {"headers": {"name": "value", ...}, "body": "..."}, replaces values +/// whose header NAME (case-insensitive) is in +/// with "<redacted>", +/// and re-serialises. Non-JSON inputs pass through unchanged (no-op for +/// emitters that have not yet adopted the convention). The stage runs BEFORE +/// truncation so the redaction marker survives the cap. +/// +public class HeaderRedactionTests +{ + private static IOptionsMonitor Monitor(AuditLogOptions? opts = null) => + new StaticMonitor(opts ?? new AuditLogOptions()); + + private static DefaultAuditPayloadFilter Filter(AuditLogOptions? opts = null) => + new(Monitor(opts), NullLogger.Instance); + + private static AuditEvent NewEvent( + AuditStatus status = AuditStatus.Delivered, + string? request = null, + string? response = null) => new() + { + EventId = Guid.NewGuid(), + OccurredAtUtc = DateTime.UtcNow, + Channel = AuditChannel.ApiOutbound, + Kind = AuditKind.ApiCall, + Status = status, + RequestSummary = request, + ResponseSummary = response, + }; + + private static string BuildSummary(IDictionary headers, string body) + { + // Serialize via System.Text.Json so we get a representative shape. + return JsonSerializer.Serialize(new + { + headers = headers, + body = body, + }); + } + + private static IDictionary ParseSummary(string? summary) + { + Assert.NotNull(summary); + using var doc = JsonDocument.Parse(summary!); + var dict = new Dictionary(); + foreach (var property in doc.RootElement.EnumerateObject()) + { + dict[property.Name] = property.Value.Clone(); + } + return dict; + } + + [Fact] + public void HeaderRedaction_AuthorizationBearer_Redacted() + { + var headers = new Dictionary + { + ["Authorization"] = "Bearer secret-token-xyz", + ["Content-Type"] = "application/json", + }; + var input = BuildSummary(headers, "hello"); + var evt = NewEvent(request: input); + + var result = Filter().Apply(evt); + + var parsed = ParseSummary(result.RequestSummary); + var resultHeaders = parsed["headers"]; + Assert.Equal("", resultHeaders.GetProperty("Authorization").GetString()); + } + + [Fact] + public void HeaderRedaction_CaseInsensitive_LowercaseAuthorization_Redacted() + { + var headers = new Dictionary + { + ["authorization"] = "Bearer secret-token-xyz", + }; + var input = BuildSummary(headers, "hello"); + var evt = NewEvent(request: input); + + var result = Filter().Apply(evt); + + var parsed = ParseSummary(result.RequestSummary); + var resultHeaders = parsed["headers"]; + Assert.Equal("", resultHeaders.GetProperty("authorization").GetString()); + } + + [Fact] + public void HeaderRedaction_CustomRedactList_RedactsCustomHeaderName() + { + var opts = new AuditLogOptions + { + HeaderRedactList = new List { "X-Custom-Secret" }, + }; + var headers = new Dictionary + { + ["X-Custom-Secret"] = "topsecret", + ["Authorization"] = "Bearer keep-me", // not in list anymore + }; + var input = BuildSummary(headers, "hi"); + var evt = NewEvent(request: input); + + var result = Filter(opts).Apply(evt); + + var parsed = ParseSummary(result.RequestSummary); + var resultHeaders = parsed["headers"]; + Assert.Equal("", resultHeaders.GetProperty("X-Custom-Secret").GetString()); + // Authorization no longer listed -> preserved verbatim. + Assert.Equal("Bearer keep-me", resultHeaders.GetProperty("Authorization").GetString()); + } + + [Fact] + public void HeaderRedaction_NonJson_RequestSummary_Unchanged() + { + const string input = "this is not JSON at all"; + var evt = NewEvent(request: input); + + var result = Filter().Apply(evt); + + Assert.Equal(input, result.RequestSummary); + } + + [Fact] + public void HeaderRedaction_NoHeadersField_Unchanged() + { + var input = JsonSerializer.Serialize(new { body = "only a body, no headers" }); + var evt = NewEvent(request: input); + + var result = Filter().Apply(evt); + + // The stage may re-serialise but the content must be semantically identical. + var parsed = ParseSummary(result.RequestSummary); + Assert.Equal("only a body, no headers", parsed["body"].GetString()); + Assert.False(parsed.ContainsKey("headers")); + } + + [Fact] + public void HeaderRedaction_Other_Headers_Preserved() + { + var headers = new Dictionary + { + ["Authorization"] = "Bearer secret", + ["Content-Type"] = "application/json", + ["X-Request-Id"] = "abc-123", + ["Accept"] = "application/json", + }; + var input = BuildSummary(headers, "payload"); + var evt = NewEvent(request: input); + + var result = Filter().Apply(evt); + + var parsed = ParseSummary(result.RequestSummary); + var resultHeaders = parsed["headers"]; + Assert.Equal("", resultHeaders.GetProperty("Authorization").GetString()); + Assert.Equal("application/json", resultHeaders.GetProperty("Content-Type").GetString()); + Assert.Equal("abc-123", resultHeaders.GetProperty("X-Request-Id").GetString()); + Assert.Equal("application/json", resultHeaders.GetProperty("Accept").GetString()); + } + + [Fact] + public void HeaderRedaction_AppliedBeforeTruncation() + { + // Build a summary whose Authorization header value is enormous AND whose + // body padding pushes the total beyond the 8 KB cap. After redaction the + // Authorization value becomes "" — then truncation caps the + // re-serialised string. Result must: + // * carry "" (header redaction ran first), + // * NOT carry the original secret bytes (proves redaction won, not order swap), + // * be capped at the configured DefaultCapBytes, + // * have PayloadTruncated == true. + const string secret = "SUPER-SECRET-TOKEN-DO-NOT-LEAK"; + var headers = new Dictionary + { + ["Authorization"] = "Bearer " + secret, + }; + var body = new string('x', 9 * 1024); + var input = BuildSummary(headers, body); + Assert.True(Encoding.UTF8.GetByteCount(input) > 8192); + + var evt = NewEvent(AuditStatus.Delivered, request: input); + + var result = Filter().Apply(evt); + + Assert.NotNull(result.RequestSummary); + Assert.True(Encoding.UTF8.GetByteCount(result.RequestSummary!) <= 8192); + Assert.Contains("", result.RequestSummary); + Assert.DoesNotContain(secret, result.RequestSummary); + Assert.True(result.PayloadTruncated); + } + + /// + /// IOptionsMonitor test double — returns the same snapshot on every read, + /// no change-token plumbing required for these tests. + /// + private sealed class StaticMonitor : IOptionsMonitor + { + private readonly AuditLogOptions _value; + public StaticMonitor(AuditLogOptions value) => _value = value; + public AuditLogOptions CurrentValue => _value; + public AuditLogOptions Get(string? name) => _value; + public IDisposable? OnChange(Action listener) => null; + } +} diff --git a/tests/ScadaLink.AuditLog.Tests/Payload/PayloadFilterContractTests.cs b/tests/ScadaLink.AuditLog.Tests/Payload/PayloadFilterContractTests.cs new file mode 100644 index 0000000..6848b29 --- /dev/null +++ b/tests/ScadaLink.AuditLog.Tests/Payload/PayloadFilterContractTests.cs @@ -0,0 +1,59 @@ +using System.Linq; +using System.Reflection; +using ScadaLink.AuditLog.Payload; +using ScadaLink.Commons.Entities.Audit; + +namespace ScadaLink.AuditLog.Tests.Payload; + +/// +/// Bundle A (M5-T1) contract test for . The +/// interface is the seam between event construction and writer persistence; +/// later bundles register the production implementation as a singleton and +/// invoke it from the site/central writer paths. We pin the surface area here +/// via reflection so accidental signature drift breaks the build before the +/// downstream wiring goes red. +/// +public class PayloadFilterContractTests +{ + [Fact] + public void Interface_Exists_InPayloadNamespace() + { + var type = typeof(IAuditPayloadFilter); + + Assert.True(type.IsInterface, "IAuditPayloadFilter must be an interface"); + Assert.Equal("ScadaLink.AuditLog.Payload", type.Namespace); + } + + [Fact] + public void Apply_Method_HasDocumentedSignature() + { + var type = typeof(IAuditPayloadFilter); + + var method = type.GetMethod( + "Apply", + BindingFlags.Instance | BindingFlags.Public, + binder: null, + types: new[] { typeof(AuditEvent) }, + modifiers: null); + + Assert.NotNull(method); + Assert.Equal(typeof(AuditEvent), method!.ReturnType); + + var parameters = method.GetParameters(); + Assert.Single(parameters); + Assert.Equal("rawEvent", parameters[0].Name); + Assert.Equal(typeof(AuditEvent), parameters[0].ParameterType); + } + + [Fact] + public void Interface_DeclaresExactlyOneMethod() + { + var type = typeof(IAuditPayloadFilter); + var methods = type.GetMethods(BindingFlags.Instance | BindingFlags.Public) + .Where(m => !m.IsSpecialName) + .ToArray(); + + Assert.Single(methods); + Assert.Equal("Apply", methods[0].Name); + } +} diff --git a/tests/ScadaLink.AuditLog.Tests/Payload/RedactionSafetyNetTests.cs b/tests/ScadaLink.AuditLog.Tests/Payload/RedactionSafetyNetTests.cs new file mode 100644 index 0000000..9ebddb0 --- /dev/null +++ b/tests/ScadaLink.AuditLog.Tests/Payload/RedactionSafetyNetTests.cs @@ -0,0 +1,270 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using ScadaLink.AuditLog.Configuration; +using ScadaLink.AuditLog.Payload; +using ScadaLink.Commons.Entities.Audit; +using ScadaLink.Commons.Types.Enums; + +namespace ScadaLink.AuditLog.Tests.Payload; + +/// +/// Bundle D (M5-T10) safety-net edge cases for +/// . Bundle B already pinned the +/// happy-path safety net (catastrophic-backtracking timeout → +/// <redacted: redactor error> + counter bump); this fixture covers +/// the pathological / config-mistake corners that production operators will +/// hit when typoing a regex or shipping a half-baked redactor list. +/// +/// +/// +/// The invariants under test: +/// +/// +/// An UNCOMPILABLE pattern (e.g. [unclosed) is logged at warning +/// on first encounter and cached as invalid so it never throws again, +/// but the redactor-failure COUNTER is not bumped at bind time — +/// per the contract on +/// the counter tracks RUNTIME redaction failures only. +/// One throwing regex in the middle of a list does NOT poison the +/// other patterns — the filter stops at the failing pattern, +/// over-redacts the offending field, but lets every other field keep +/// the prior cleanly-redacted state and lets the rest of the writer +/// pipeline run. +/// A live config change that introduces a broken pattern does not +/// crash the filter — the bad pattern is silently dropped (logged once) +/// and the still-valid patterns continue to redact normally. +/// +/// +public class RedactionSafetyNetTests +{ + private static IOptionsMonitor Monitor(AuditLogOptions? opts = null) => + new StaticMonitor(opts ?? new AuditLogOptions()); + + private static AuditEvent NewEvent( + AuditStatus status = AuditStatus.Delivered, + string? request = null, + string? response = null, + string? errorDetail = null, + string? extra = null, + string? target = null) => new() + { + EventId = Guid.NewGuid(), + OccurredAtUtc = DateTime.UtcNow, + Channel = AuditChannel.ApiOutbound, + Kind = AuditKind.ApiCall, + Status = status, + Target = target, + RequestSummary = request, + ResponseSummary = response, + ErrorDetail = errorDetail, + Extra = extra, + }; + + [Fact] + public void RegexNotCompilable_AtBindTime_LoggedAndSkipped() + { + // `[unclosed` is a structurally invalid character class — the .NET + // regex engine throws ArgumentException at compile time. We assert: + // * the filter does NOT throw, + // * the OTHER (valid) pattern still redacts hunter2, + // * the failure counter is NOT incremented at compile time + // (it tracks runtime redaction failures only), + // * a warning is logged exactly once. + const string badPattern = "[unclosed"; + const string goodPattern = "\"password\":\\s*\"[^\"]*\""; + var opts = new AuditLogOptions + { + GlobalBodyRedactors = new List { badPattern, goodPattern }, + }; + var counter = new CountingRedactionFailureCounter(); + var spy = new SpyLogger(); + var filter = new DefaultAuditPayloadFilter(Monitor(opts), spy, counter); + + var evt = NewEvent(request: "{\"user\":\"alice\",\"password\":\"hunter2\"}"); + + var result = filter.Apply(evt); + + Assert.NotNull(result.RequestSummary); + Assert.DoesNotContain("hunter2", result.RequestSummary); + Assert.Contains("", result.RequestSummary); + Assert.Equal(0, counter.Count); + // Apply twice — the invalid-pattern compile must run AT MOST once; + // the sentinel-cache entry stops repeat compile attempts. + _ = filter.Apply(evt); + var badPatternWarnings = spy.Entries + .Where(e => e.Level == LogLevel.Warning && e.Message.Contains(badPattern)) + .Count(); + Assert.Equal(1, badPatternWarnings); + } + + [Fact] + public void MultipleRedactors_OneThrows_OthersStillApply_ToOtherFields() + { + // Pattern set: [valid-A, evil, valid-B]. The evil pattern is + // catastrophic-backtracking on the RequestSummary input (all-'a's + + // mismatching suffix) — that field is over-redacted with the error + // marker as soon as evil throws. ResponseSummary is processed + // INDEPENDENTLY; its input does not trigger evil's backtracking, so + // valid-A and valid-B both still apply on that field. This proves a + // per-field redactor failure does not poison the rest of the writer + // call (the SQL-param stage, the truncation stage, and the other + // fields all continue normally). + const string validA = "SECRET-[A-Z0-9]+"; + const string evil = "^(a+)+$"; // catastrophic on long all-'a' string + const string validB = "PIN-\\d{4}"; + var opts = new AuditLogOptions + { + GlobalBodyRedactors = new List { validA, evil, validB }, + }; + var counter = new CountingRedactionFailureCounter(); + var filter = new DefaultAuditPayloadFilter( + Monitor(opts), + NullLogger.Instance, + counter); + + // Request: ALL 'a's + a non-'a' suffix character. valid-A does not + // match (no SECRET-X prefix), so the buffer reaches `evil` untouched + // and triggers the backtracking explosion. + var request = new string('a', 30) + "!"; + // Response: short, mismatches the evil pattern cleanly (no + // backtracking), so both valid-A and valid-B run and redact. + const string response = "SECRET-ABC456 PIN-9999 other-text"; + + var result = filter.Apply(NewEvent(request: request, response: response)); + + // RequestSummary: over-redacted (evil pattern threw). + Assert.Equal("", result.RequestSummary); + Assert.True(counter.Count >= 1, $"expected counter >= 1, got {counter.Count}"); + + // ResponseSummary: clean — both valid regexes still applied; the evil + // one ran without throwing on this short input. + Assert.NotNull(result.ResponseSummary); + Assert.DoesNotContain("SECRET-ABC456", result.ResponseSummary); + Assert.DoesNotContain("PIN-9999", result.ResponseSummary); + Assert.Contains("", result.ResponseSummary); + Assert.Contains("other-text", result.ResponseSummary); + } + + // Edge case 3 (RedactorReturnsNonStringExceptionType) intentionally + // skipped — the brief permits dropping it: there is no portable way to + // artificially trigger an OutOfMemoryException inside System.Text.RegularExpressions + // from a unit test without writing native interop, and the existing + // per-stage try/catch already covers Exception (which OOM and similar + // would derive from). Bundle B's RegexThrowsTimeout coverage exercises + // the same catch path with a deterministic trigger. + + [Fact] + public void ConfigChange_WithBadRegex_LiveTrafficKeepsApplyingValidRegexes() + { + // Initial config: one valid global redactor — hunter2 is redacted. + // Reload: ADD a malformed pattern alongside the original. Per the + // safety contract, the bad pattern is logged + skipped, the original + // valid pattern keeps redacting, and the filter NEVER throws on the + // hot path. The counter must not be bumped at reload time (the + // CompiledRegex sentinel covers the bind error before runtime even + // sees it). + var monitor = new MutableMonitor(new AuditLogOptions + { + GlobalBodyRedactors = new List { "\"password\":\\s*\"[^\"]*\"" }, + }); + var counter = new CountingRedactionFailureCounter(); + var spy = new SpyLogger(); + var filter = new DefaultAuditPayloadFilter(monitor, spy, counter); + + var evt = NewEvent(request: "{\"user\":\"alice\",\"password\":\"hunter2\"}"); + + var before = filter.Apply(evt); + Assert.DoesNotContain("hunter2", before.RequestSummary!); + + // Reload: malformed pattern added to the list. + monitor.Set(new AuditLogOptions + { + GlobalBodyRedactors = new List + { + "\"password\":\\s*\"[^\"]*\"", + "[unclosed", + }, + }); + + var after = filter.Apply(evt); + Assert.NotNull(after.RequestSummary); + Assert.DoesNotContain("hunter2", after.RequestSummary); + Assert.Contains("", after.RequestSummary); + Assert.Equal(0, counter.Count); + // Compile-time warning logged for the broken pattern. + Assert.Contains( + spy.Entries, + e => e.Level == LogLevel.Warning && e.Message.Contains("[unclosed")); + } + + /// Counts calls. + private sealed class CountingRedactionFailureCounter : IAuditRedactionFailureCounter + { + private int _count; + public int Count => _count; + public void Increment() => System.Threading.Interlocked.Increment(ref _count); + } + + /// IOptionsMonitor test double — returns the same snapshot on every read. + private sealed class StaticMonitor : IOptionsMonitor + { + private readonly AuditLogOptions _value; + public StaticMonitor(AuditLogOptions value) => _value = value; + public AuditLogOptions CurrentValue => _value; + public AuditLogOptions Get(string? name) => _value; + public IDisposable? OnChange(Action listener) => null; + } + + /// + /// IOptionsMonitor test double that supports a live — + /// mirrors the helper used in + /// ; + /// kept private here so the safety-net test file remains self-contained. + /// + private sealed class MutableMonitor : IOptionsMonitor + { + private AuditLogOptions _current; + public MutableMonitor(AuditLogOptions initial) => _current = initial; + public AuditLogOptions CurrentValue => _current; + public AuditLogOptions Get(string? name) => _current; + public IDisposable? OnChange(Action listener) => null; + public void Set(AuditLogOptions value) => _current = value; + } + + /// + /// Minimal ILogger that records each formatted log line so tests can + /// assert on the compile-time warning emission contract — counting + /// warnings and grepping the message text. + /// + private sealed class SpyLogger : ILogger + { + private readonly List _entries = new(); + + public IReadOnlyList Entries + { + get { lock (_entries) return _entries.ToArray(); } + } + + public IDisposable BeginScope(TState state) where TState : notnull => NullScope.Instance; + public bool IsEnabled(LogLevel logLevel) => true; + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + var msg = formatter(state, exception); + lock (_entries) _entries.Add(new LogEntry(logLevel, msg)); + } + + private sealed class NullScope : IDisposable + { + public static readonly NullScope Instance = new(); + public void Dispose() { } + } + } + + public sealed record LogEntry(LogLevel Level, string Message); +} diff --git a/tests/ScadaLink.AuditLog.Tests/Payload/SqlParamRedactionTests.cs b/tests/ScadaLink.AuditLog.Tests/Payload/SqlParamRedactionTests.cs new file mode 100644 index 0000000..0676ab8 --- /dev/null +++ b/tests/ScadaLink.AuditLog.Tests/Payload/SqlParamRedactionTests.cs @@ -0,0 +1,212 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using ScadaLink.AuditLog.Configuration; +using ScadaLink.AuditLog.Payload; +using ScadaLink.Commons.Entities.Audit; +using ScadaLink.Commons.Types.Enums; + +namespace ScadaLink.AuditLog.Tests.Payload; + +/// +/// Bundle B (M5-T5) tests for SQL parameter redaction in +/// . M4 Bundle A's +/// AuditingDbCommand emits RequestSummary as +/// {"sql":"...","parameters":{"@name":"value", ...}}; the SQL-parameter +/// redactor parses this shape on +/// rows, replaces values whose key +/// matches the configured case-insensitive regex with <redacted>, +/// and re-serialises. Default behaviour with no opt-in: parameter values are +/// captured verbatim. Connection lookup uses the connection-name prefix of +/// (everything before the first .) so +/// the same per-connection regex applies regardless of the SQL-snippet suffix +/// that AuditingDbCommand appends to disambiguate rows. +/// +public class SqlParamRedactionTests +{ + private static IOptionsMonitor Monitor(AuditLogOptions? opts = null) => + new StaticMonitor(opts ?? new AuditLogOptions()); + + private static DefaultAuditPayloadFilter Filter(AuditLogOptions? opts = null) => + new(Monitor(opts), NullLogger.Instance); + + private static AuditEvent NewDbEvent(string target, string requestSummary) => new() + { + EventId = Guid.NewGuid(), + OccurredAtUtc = DateTime.UtcNow, + Channel = AuditChannel.DbOutbound, + Kind = AuditKind.DbWrite, + Status = AuditStatus.Delivered, + Target = target, + RequestSummary = requestSummary, + }; + + /// + /// Build a RequestSummary in the exact shape M4's AuditingDbCommand + /// emits — hand-rolled JSON with "sql" + "parameters" keys. + /// Tests depend on this format; if AuditingDbCommand ever changes, this + /// helper updates in lockstep. + /// + private static string DbRequestSummary(string sql, params (string name, string value)[] parameters) + { + var sb = new System.Text.StringBuilder(); + sb.Append("{\"sql\":\"").Append(sql).Append('"'); + if (parameters.Length > 0) + { + sb.Append(",\"parameters\":{"); + for (var i = 0; i < parameters.Length; i++) + { + if (i > 0) sb.Append(','); + sb.Append('"').Append(parameters[i].name).Append("\":\"") + .Append(parameters[i].value).Append('"'); + } + sb.Append('}'); + } + sb.Append('}'); + return sb.ToString(); + } + + [Fact] + public void NoOptIn_ParamsVerbatim_Unchanged() + { + var input = DbRequestSummary( + "INSERT INTO Users (Name, Token) VALUES (@name, @token)", + ("@name", "Alice"), + ("@token", "secret-xyz")); + var evt = NewDbEvent("PrimaryDb.INSERT INTO Users", input); + + var result = Filter().Apply(evt); + + Assert.Equal(input, result.RequestSummary); + } + + [Fact] + public void OptInRegex_AtToken_OrAtApikey_RedactsThoseValues_KeepsOthers() + { + var opts = new AuditLogOptions + { + PerTargetOverrides = new Dictionary + { + ["PrimaryDb"] = new PerTargetRedactionOverride + { + RedactSqlParamsMatching = "^@(token|apikey)$", + }, + }, + }; + var input = DbRequestSummary( + "INSERT INTO Users (Name, Token, ApiKey) VALUES (@name, @token, @apikey)", + ("@name", "Alice"), + ("@token", "secret-xyz"), + ("@apikey", "k-987")); + var evt = NewDbEvent("PrimaryDb.INSERT INTO Users", input); + + var result = Filter(opts).Apply(evt); + + Assert.NotNull(result.RequestSummary); + Assert.Contains("\"@name\":\"Alice\"", result.RequestSummary); + Assert.Contains("\"@token\":\"\"", result.RequestSummary); + Assert.Contains("\"@apikey\":\"\"", result.RequestSummary); + Assert.DoesNotContain("secret-xyz", result.RequestSummary); + Assert.DoesNotContain("k-987", result.RequestSummary); + } + + [Fact] + public void RegexCaseInsensitive_MatchesParamNames() + { + var opts = new AuditLogOptions + { + PerTargetOverrides = new Dictionary + { + ["PrimaryDb"] = new PerTargetRedactionOverride + { + RedactSqlParamsMatching = "token", + }, + }, + }; + var input = DbRequestSummary( + "UPDATE x SET Token = @TOKEN", + ("@TOKEN", "uppercased-secret")); + var evt = NewDbEvent("PrimaryDb.UPDATE x SET Token", input); + + var result = Filter(opts).Apply(evt); + + Assert.NotNull(result.RequestSummary); + Assert.Contains("\"@TOKEN\":\"\"", result.RequestSummary); + Assert.DoesNotContain("uppercased-secret", result.RequestSummary); + } + + [Fact] + public void NonDbOutboundChannel_NotAffected() + { + // ApiOutbound row whose RequestSummary happens to look like the + // DbOutbound JSON shape (worst-case false positive). The SQL + // redactor must NOT touch it — channel guards the stage. + var opts = new AuditLogOptions + { + PerTargetOverrides = new Dictionary + { + ["PrimaryDb"] = new PerTargetRedactionOverride + { + RedactSqlParamsMatching = "^@token$", + }, + }, + }; + var input = DbRequestSummary( + "SELECT @token", + ("@token", "should-survive")); + var evt = new AuditEvent + { + EventId = Guid.NewGuid(), + OccurredAtUtc = DateTime.UtcNow, + Channel = AuditChannel.ApiOutbound, + Kind = AuditKind.ApiCall, + Status = AuditStatus.Delivered, + Target = "PrimaryDb.SELECT", // doesn't matter — channel guards + RequestSummary = input, + }; + + var result = Filter(opts).Apply(evt); + + Assert.Equal(input, result.RequestSummary); + } + + [Fact] + public void PerTargetSetting_MatchesByTarget() + { + // Two connections — A is configured to redact tokens, B is not. Same + // payload through each must yield different results. + var opts = new AuditLogOptions + { + PerTargetOverrides = new Dictionary + { + ["ConnA"] = new PerTargetRedactionOverride + { + RedactSqlParamsMatching = "^@token$", + }, + }, + }; + var input = DbRequestSummary( + "SELECT @token", + ("@token", "the-secret")); + + var aEvt = NewDbEvent("ConnA.SELECT @token", input); + var bEvt = NewDbEvent("ConnB.SELECT @token", input); + + var aResult = Filter(opts).Apply(aEvt); + var bResult = Filter(opts).Apply(bEvt); + + Assert.Contains("", aResult.RequestSummary!); + Assert.DoesNotContain("the-secret", aResult.RequestSummary!); + + Assert.Equal(input, bResult.RequestSummary); + } + + /// IOptionsMonitor test double. + private sealed class StaticMonitor : IOptionsMonitor + { + private readonly AuditLogOptions _value; + public StaticMonitor(AuditLogOptions value) => _value = value; + public AuditLogOptions CurrentValue => _value; + public AuditLogOptions Get(string? name) => _value; + public IDisposable? OnChange(Action listener) => null; + } +} diff --git a/tests/ScadaLink.AuditLog.Tests/Payload/TruncationTests.cs b/tests/ScadaLink.AuditLog.Tests/Payload/TruncationTests.cs new file mode 100644 index 0000000..d747336 --- /dev/null +++ b/tests/ScadaLink.AuditLog.Tests/Payload/TruncationTests.cs @@ -0,0 +1,226 @@ +using System.Linq; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using ScadaLink.AuditLog.Configuration; +using ScadaLink.AuditLog.Payload; +using ScadaLink.Commons.Entities.Audit; +using ScadaLink.Commons.Types.Enums; + +namespace ScadaLink.AuditLog.Tests.Payload; + +/// +/// Bundle A (M5-T2) tests for truncation. +/// The filter caps RequestSummary / ResponseSummary / ErrorDetail / Extra at +/// (8 KiB) on success rows and +/// (64 KiB) on error rows. "Error +/// row" = NOT IN (Delivered, +/// Submitted, Forwarded). Truncation must respect UTF-8 character +/// boundaries (never split a multi-byte sequence mid-character) and must set +/// true when any field is shortened. +/// +public class TruncationTests +{ + private static IOptionsMonitor Monitor(AuditLogOptions? opts = null) + { + var snapshot = opts ?? new AuditLogOptions(); + return new StaticMonitor(snapshot); + } + + private static DefaultAuditPayloadFilter Filter(AuditLogOptions? opts = null) => + new(Monitor(opts), NullLogger.Instance); + + private static AuditEvent NewEvent( + AuditStatus status = AuditStatus.Delivered, + string? request = null, + string? response = null, + string? errorDetail = null, + string? extra = null, + bool payloadTruncated = false) => new() + { + EventId = Guid.NewGuid(), + OccurredAtUtc = DateTime.UtcNow, + Channel = AuditChannel.ApiOutbound, + Kind = AuditKind.ApiCall, + Status = status, + RequestSummary = request, + ResponseSummary = response, + ErrorDetail = errorDetail, + Extra = extra, + PayloadTruncated = payloadTruncated, + }; + + [Fact] + public void SuccessRow_10KB_RequestSummary_TruncatedTo8KB_PayloadTruncatedTrue() + { + var input = new string('a', 10 * 1024); + var evt = NewEvent(AuditStatus.Delivered, request: input); + + var result = Filter().Apply(evt); + + Assert.NotNull(result.RequestSummary); + Assert.Equal(8192, Encoding.UTF8.GetByteCount(result.RequestSummary!)); + Assert.True(result.PayloadTruncated); + } + + [Fact] + public void ErrorRow_10KB_RequestSummary_NotTruncated_UnderErrorCap() + { + var input = new string('b', 10 * 1024); + var evt = NewEvent(AuditStatus.Failed, request: input); + + var result = Filter().Apply(evt); + + Assert.Equal(input, result.RequestSummary); + Assert.False(result.PayloadTruncated); + } + + [Fact] + public void ErrorRow_70KB_RequestSummary_TruncatedTo64KB_PayloadTruncatedTrue() + { + var input = new string('c', 70 * 1024); + var evt = NewEvent(AuditStatus.Failed, request: input); + + var result = Filter().Apply(evt); + + Assert.NotNull(result.RequestSummary); + Assert.Equal(65536, Encoding.UTF8.GetByteCount(result.RequestSummary!)); + Assert.True(result.PayloadTruncated); + } + + [Fact] + public void Multibyte_UTF8_TruncatedAtCharacterBoundary_NotMidByte() + { + // U+1F600 (grinning face) encodes to 4 UTF-8 bytes; 2000 of them = 8000 bytes, + // safely under the 8192 default cap so the boundary scan kicks in mid-character + // when we push past it. Pad with a few extra emoji so the *input* is > 8192 bytes + // and forces truncation. + var emoji = "😀"; // surrogate pair => one code point => 4 UTF-8 bytes + var sb = new StringBuilder(); + for (int i = 0; i < 2100; i++) + { + sb.Append(emoji); + } + var input = sb.ToString(); + Assert.True(Encoding.UTF8.GetByteCount(input) > 8192); + + var evt = NewEvent(AuditStatus.Delivered, request: input); + + var result = Filter().Apply(evt); + + Assert.NotNull(result.RequestSummary); + var resultBytes = Encoding.UTF8.GetByteCount(result.RequestSummary!); + Assert.True(resultBytes <= 8192, $"expected <= 8192 bytes, got {resultBytes}"); + // 4-byte emoji boundary: the kept byte length must be a multiple of 4. + Assert.Equal(0, resultBytes % 4); + // And round-tripping the result must not introduce a U+FFFD replacement char. + Assert.DoesNotContain('�', result.RequestSummary); + Assert.True(result.PayloadTruncated); + } + + [Fact] + public void NullSummary_PassesThrough_AsNull() + { + var evt = NewEvent(AuditStatus.Delivered, request: null, response: null, errorDetail: null, extra: null); + + var result = Filter().Apply(evt); + + Assert.Null(result.RequestSummary); + Assert.Null(result.ResponseSummary); + Assert.Null(result.ErrorDetail); + Assert.Null(result.Extra); + Assert.False(result.PayloadTruncated); + } + + [Fact] + public void RawEventAlreadyTruncated_PayloadTruncatedRemainsTrue() + { + // Small payload that requires no truncation, but the caller already + // flagged PayloadTruncated upstream — the filter must not clear it. + var evt = NewEvent(AuditStatus.Delivered, request: "small", payloadTruncated: true); + + var result = Filter().Apply(evt); + + Assert.Equal("small", result.RequestSummary); + Assert.True(result.PayloadTruncated); + } + + [Fact] + public void StatusAttempted_TreatedAsError_UsesErrorCap() + { + // 10 KB is under the 64 KB error cap; if Attempted were a success status + // the value would be truncated to 8 KB. We assert it is NOT truncated. + var input = new string('d', 10 * 1024); + var evt = NewEvent(AuditStatus.Attempted, request: input); + + var result = Filter().Apply(evt); + + Assert.Equal(input, result.RequestSummary); + Assert.False(result.PayloadTruncated); + } + + [Fact] + public void StatusParked_TreatedAsError_UsesErrorCap() + { + var input = new string('e', 10 * 1024); + var evt = NewEvent(AuditStatus.Parked, request: input); + + var result = Filter().Apply(evt); + + Assert.Equal(input, result.RequestSummary); + Assert.False(result.PayloadTruncated); + } + + [Fact] + public void StatusSkipped_TreatedAsError_UsesErrorCap() + { + var input = new string('f', 10 * 1024); + var evt = NewEvent(AuditStatus.Skipped, request: input); + + var result = Filter().Apply(evt); + + Assert.Equal(input, result.RequestSummary); + Assert.False(result.PayloadTruncated); + } + + [Fact] + public void ErrorDetail_AndExtra_Truncated_Independently() + { + // Each field is capped on its own — a 10 KB RequestSummary and a 10 KB + // ErrorDetail on the same Delivered row should both be cut to 8 KB and + // the row flagged truncated. + var input = new string('g', 10 * 1024); + var evt = NewEvent( + AuditStatus.Delivered, + request: input, + response: input, + errorDetail: input, + extra: input); + + var result = Filter().Apply(evt); + + Assert.Equal(8192, Encoding.UTF8.GetByteCount(result.RequestSummary!)); + Assert.Equal(8192, Encoding.UTF8.GetByteCount(result.ResponseSummary!)); + Assert.Equal(8192, Encoding.UTF8.GetByteCount(result.ErrorDetail!)); + Assert.Equal(8192, Encoding.UTF8.GetByteCount(result.Extra!)); + Assert.True(result.PayloadTruncated); + } + + /// + /// IOptionsMonitor test double — returns the same snapshot on every read, + /// no change-token plumbing required for these tests (Bundle D wires the + /// real hot-reload path). + /// + private sealed class StaticMonitor : IOptionsMonitor + { + private readonly AuditLogOptions _value; + + public StaticMonitor(AuditLogOptions value) => _value = value; + + public AuditLogOptions CurrentValue => _value; + + public AuditLogOptions Get(string? name) => _value; + + public IDisposable? OnChange(Action listener) => null; + } +} diff --git a/tests/ScadaLink.AuditLog.Tests/Site/HealthMetricsAuditRedactionFailureCounterTests.cs b/tests/ScadaLink.AuditLog.Tests/Site/HealthMetricsAuditRedactionFailureCounterTests.cs new file mode 100644 index 0000000..ffc09f3 --- /dev/null +++ b/tests/ScadaLink.AuditLog.Tests/Site/HealthMetricsAuditRedactionFailureCounterTests.cs @@ -0,0 +1,49 @@ +using NSubstitute; +using ScadaLink.AuditLog.Site; +using ScadaLink.HealthMonitoring; + +namespace ScadaLink.AuditLog.Tests.Site; + +/// +/// Bundle C (M5-T7) — the +/// adapter is the production binding for +/// on +/// site nodes; it forwards every +/// redactor over-redaction event into the shared +/// so the site health report surfaces the +/// count as AuditRedactionFailure. Mirrors the M2 Bundle G +/// HealthMetricsAuditWriteFailureCounter shape one-for-one. +/// +public class HealthMetricsAuditRedactionFailureCounterTests +{ + [Fact] + public void Increment_Routes_To_Collector_IncrementAuditRedactionFailure() + { + var collector = Substitute.For(); + var counter = new HealthMetricsAuditRedactionFailureCounter(collector); + + counter.Increment(); + + collector.Received(1).IncrementAuditRedactionFailure(); + } + + [Fact] + public void Increment_Multiple_Calls_Route_To_Collector_Each_Time() + { + var collector = Substitute.For(); + var counter = new HealthMetricsAuditRedactionFailureCounter(collector); + + counter.Increment(); + counter.Increment(); + counter.Increment(); + + collector.Received(3).IncrementAuditRedactionFailure(); + } + + [Fact] + public void Construction_With_Null_Collector_Throws_ArgumentNullException() + { + Assert.Throws( + () => new HealthMetricsAuditRedactionFailureCounter(null!)); + } +} diff --git a/tests/ScadaLink.HealthMonitoring.Tests/AuditRedactionFailureMetricTests.cs b/tests/ScadaLink.HealthMonitoring.Tests/AuditRedactionFailureMetricTests.cs new file mode 100644 index 0000000..2618561 --- /dev/null +++ b/tests/ScadaLink.HealthMonitoring.Tests/AuditRedactionFailureMetricTests.cs @@ -0,0 +1,57 @@ +namespace ScadaLink.HealthMonitoring.Tests; + +/// +/// Bundle C (M5-T7) regression coverage. The Audit Log payload filter +/// (DefaultAuditPayloadFilter) increments +/// IAuditRedactionFailureCounter every time a header/body/SQL-param +/// redactor stage throws and the filter has to over-redact the field with +/// the <redacted: redactor error> marker. Bundle C bridges that +/// counter into the Site Health Monitoring report payload as +/// AuditRedactionFailure so a misconfigured / catastrophic regex +/// surfaces on /monitoring/health rather than disappearing into a NoOp sink. +/// Mirrors the Bundle G SiteAuditWriteFailures metric shape — same +/// per-interval increment-and-reset semantics, same defaults-to-zero +/// contract. +/// +public class AuditRedactionFailureMetricTests +{ + private readonly SiteHealthCollector _collector = new(); + + [Fact] + public void Increment_Three_Times_Counter_Reports_3() + { + _collector.IncrementAuditRedactionFailure(); + _collector.IncrementAuditRedactionFailure(); + _collector.IncrementAuditRedactionFailure(); + + var report = _collector.CollectReport("site-1"); + + Assert.Equal(3, report.AuditRedactionFailure); + } + + [Fact] + public void Report_Payload_Includes_AuditRedactionFailure_AsZeroByDefault() + { + var report = _collector.CollectReport("site-1"); + + Assert.Equal(0, report.AuditRedactionFailure); + } + + /// + /// Mirrors the existing per-interval reset semantics for ScriptErrorCount / + /// AlarmEvaluationErrorCount / DeadLetterCount / SiteAuditWriteFailures — + /// AuditRedactionFailure is an interval count, not a running total. + /// + [Fact] + public void CollectReport_Resets_AuditRedactionFailure() + { + _collector.IncrementAuditRedactionFailure(); + _collector.IncrementAuditRedactionFailure(); + + var first = _collector.CollectReport("site-1"); + Assert.Equal(2, first.AuditRedactionFailure); + + var second = _collector.CollectReport("site-1"); + Assert.Equal(0, second.AuditRedactionFailure); + } +} diff --git a/tests/ScadaLink.PerformanceTests/AuditLog/HotPathLatencyTests.cs b/tests/ScadaLink.PerformanceTests/AuditLog/HotPathLatencyTests.cs new file mode 100644 index 0000000..45b5134 --- /dev/null +++ b/tests/ScadaLink.PerformanceTests/AuditLog/HotPathLatencyTests.cs @@ -0,0 +1,163 @@ +using System.Diagnostics; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using ScadaLink.AuditLog.Configuration; +using ScadaLink.AuditLog.Payload; +using ScadaLink.Commons.Entities.Audit; +using ScadaLink.Commons.Types.Enums; + +namespace ScadaLink.PerformanceTests.AuditLog; + +/// +/// Bundle D (M5-T9) hot-path latency budget for . +/// The filter sits between event construction and persistence on every audit +/// row — site SQLite hot-path and central direct-write both — so it MUST stay +/// out of the way of script-thread latency. +/// +/// +/// +/// Methodology: warm-up + N iterations, time each +/// with , sort, take p95, assert under threshold. Matches +/// the simple-loop style of the existing StaggeredStartupTests / +/// HealthAggregationTests in this project (no BenchmarkDotNet). +/// +/// +/// Threshold note: the spec says "set during M5 brainstorm" — pick targets that +/// are an order of magnitude faster than the SQLite write they precede (the +/// site writer's bottleneck is the disk fsync, not the in-memory filter). +/// Reality may diverge on slow CI; the assertions include the empirical +/// fall-back the task brief calls for (p95 + 30% regression guard) wired +/// through environment-variable override so a slow shared runner doesn't +/// flake the build but a 10x regression still does. +/// +/// +public class HotPathLatencyTests +{ + private const int WarmupIterations = 200; + private const int MeasureIterations = 2_000; + + private static DefaultAuditPayloadFilter Filter(AuditLogOptions opts) => new( + new StaticMonitor(opts), + NullLogger.Instance); + + private static AuditEvent NewEvent(string request) + { + return new AuditEvent + { + EventId = Guid.NewGuid(), + OccurredAtUtc = DateTime.UtcNow, + Channel = AuditChannel.ApiOutbound, + Kind = AuditKind.ApiCall, + Status = AuditStatus.Delivered, + Target = "esg.target", + RequestSummary = request, + }; + } + + /// + /// Run N times, returning the p95 in microseconds. + /// Single-threaded; for high-res + /// timing. + /// + private static double MeasureP95Microseconds(int iterations, Action fn) + { + var samples = new double[iterations]; + var ticksToMicroseconds = 1_000_000d / Stopwatch.Frequency; + for (var i = 0; i < iterations; i++) + { + var start = Stopwatch.GetTimestamp(); + fn(); + var end = Stopwatch.GetTimestamp(); + samples[i] = (end - start) * ticksToMicroseconds; + } + Array.Sort(samples); + var p95Index = (int)Math.Ceiling(iterations * 0.95) - 1; + return samples[p95Index]; + } + + [Trait("Category", "Performance")] + [Fact] + public void Filter_Apply_4KB_Body_DefaultRedactors_P95_LessThan_50_Microseconds() + { + // 4 KiB body laced with a 16-digit token + a `password` field so the + // header-redact stage is a no-op (input isn't a JSON object with a + // headers field), the body regex stage matches twice, and the + // truncation stage runs after redaction. Mirrors a typical + // medium-sized HTTP POST body that an outbound API audit row would + // carry. + var opts = new AuditLogOptions + { + // Keep the cap modest so the truncation path actually fires. + DefaultCapBytes = 4096, + GlobalBodyRedactors = new List + { + "\"password\":\\s*\"[^\"]*\"", + "\\d{16}", + }, + }; + var pad = new string('x', 4 * 1024); + var body = "{\"user\":\"alice\",\"password\":\"hunter2\",\"card\":\"4111111111111111\",\"pad\":\"" + pad + "\"}"; + // Sanity: we actually want > 4 KiB so the truncate stage runs. + Assert.True(Encoding.UTF8.GetByteCount(body) > 4096); + + var filter = Filter(opts); + var evt = NewEvent(body); + + // Warm-up — JIT, regex compile, dictionary populate. + for (var i = 0; i < WarmupIterations; i++) _ = filter.Apply(evt); + + var p95Us = MeasureP95Microseconds(MeasureIterations, () => _ = filter.Apply(evt)); + + // Default budget 50 µs (spec target). Override via env for slow CI: + // SCADALINK_AUDIT_FILTER_4KB_P95_US — interpret as the regression + // guard threshold. Print the observed value so a missed budget gives + // useful telemetry on the test output. + var threshold = GetThresholdMicroseconds("SCADALINK_AUDIT_FILTER_4KB_P95_US", 50d); + Assert.True(p95Us < threshold, + $"4KB body filter p95 = {p95Us:F1} µs; threshold = {threshold:F1} µs"); + } + + [Trait("Category", "Performance")] + [Fact] + public void Filter_Apply_RawEvent_NoRedactors_P95_LessThan_10_Microseconds() + { + // No redactors configured — header redactor short-circuits on the + // non-JSON-object pre-check, body redactor list is empty, SQL param + // redactor is gated on AuditChannel.DbOutbound (we're ApiOutbound). + // Just the per-field truncation walk. Should be effectively free. + var opts = new AuditLogOptions(); + var filter = Filter(opts); + + // Small payload that fits under the 8 KiB default cap — no truncation, + // just the byte-count check per field. + var evt = NewEvent("hello world"); + + for (var i = 0; i < WarmupIterations; i++) _ = filter.Apply(evt); + + var p95Us = MeasureP95Microseconds(MeasureIterations, () => _ = filter.Apply(evt)); + + var threshold = GetThresholdMicroseconds("SCADALINK_AUDIT_FILTER_RAW_P95_US", 10d); + Assert.True(p95Us < threshold, + $"Raw-event filter p95 = {p95Us:F1} µs; threshold = {threshold:F1} µs"); + } + + private static double GetThresholdMicroseconds(string envVar, double defaultUs) + { + var raw = Environment.GetEnvironmentVariable(envVar); + if (raw != null && double.TryParse(raw, out var parsed) && parsed > 0) + { + return parsed; + } + return defaultUs; + } + + private sealed class StaticMonitor : IOptionsMonitor + { + private readonly AuditLogOptions _value; + public StaticMonitor(AuditLogOptions value) => _value = value; + public AuditLogOptions CurrentValue => _value; + public AuditLogOptions Get(string? name) => _value; + public IDisposable? OnChange(Action listener) => null; + } +} diff --git a/tests/ScadaLink.PerformanceTests/ScadaLink.PerformanceTests.csproj b/tests/ScadaLink.PerformanceTests/ScadaLink.PerformanceTests.csproj index fb6592c..659a59d 100644 --- a/tests/ScadaLink.PerformanceTests/ScadaLink.PerformanceTests.csproj +++ b/tests/ScadaLink.PerformanceTests/ScadaLink.PerformanceTests.csproj @@ -21,6 +21,7 @@ + diff --git a/tests/ScadaLink.SiteRuntime.Tests/Actors/DeploymentManagerRedeployTests.cs b/tests/ScadaLink.SiteRuntime.Tests/Actors/DeploymentManagerRedeployTests.cs index 9548631..aec5917 100644 --- a/tests/ScadaLink.SiteRuntime.Tests/Actors/DeploymentManagerRedeployTests.cs +++ b/tests/ScadaLink.SiteRuntime.Tests/Actors/DeploymentManagerRedeployTests.cs @@ -70,6 +70,7 @@ public class DeploymentManagerRedeployTests : TestKit, IDisposable public void IncrementAlarmError() { } public void IncrementDeadLetter() { } public void IncrementSiteAuditWriteFailures() { } + public void IncrementAuditRedactionFailure() { } public void UpdateConnectionHealth(string connectionName, ConnectionHealth health) { } public void RemoveConnection(string connectionName) { } public void UpdateTagResolution(string connectionName, int totalSubscribed, int successfullyResolved) { }