From adfb4d385c02ba94ceef97c11360beab8d0c2aa5 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 2 Jun 2026 11:00:36 -0400 Subject: [PATCH] =?UTF-8?q?feat(audit):=20ScadaBridge=20C2=20=E2=80=94=20S?= =?UTF-8?q?cadaBridgeAuditRedactor/SafeDefaultAuditRedactor=20:=20IAuditRe?= =?UTF-8?q?dactor=20on=20canonical=20record=20(Task=202.5)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Payload/AuditRedactionPrimitives.cs | 310 ++++++++++ .../Payload/AuditRegexCache.cs | 96 ++++ .../Payload/DefaultAuditPayloadFilter.cs | 349 ++--------- .../Redaction/SafeDefaultAuditRedactor.cs | 91 +++ .../Redaction/ScadaBridgeAuditRedactor.cs | 349 +++++++++++ .../SafeDefaultAuditRedactorTests.cs | 122 ++++ .../ScadaBridgeAuditRedactorTests.cs | 540 ++++++++++++++++++ 7 files changed, 1541 insertions(+), 316 deletions(-) create mode 100644 src/ZB.MOM.WW.ScadaBridge.AuditLog/Payload/AuditRedactionPrimitives.cs create mode 100644 src/ZB.MOM.WW.ScadaBridge.AuditLog/Payload/AuditRegexCache.cs create mode 100644 src/ZB.MOM.WW.ScadaBridge.AuditLog/Redaction/SafeDefaultAuditRedactor.cs create mode 100644 src/ZB.MOM.WW.ScadaBridge.AuditLog/Redaction/ScadaBridgeAuditRedactor.cs create mode 100644 tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Redaction/SafeDefaultAuditRedactorTests.cs create mode 100644 tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Redaction/ScadaBridgeAuditRedactorTests.cs diff --git a/src/ZB.MOM.WW.ScadaBridge.AuditLog/Payload/AuditRedactionPrimitives.cs b/src/ZB.MOM.WW.ScadaBridge.AuditLog/Payload/AuditRedactionPrimitives.cs new file mode 100644 index 00000000..663b21e5 --- /dev/null +++ b/src/ZB.MOM.WW.ScadaBridge.AuditLog/Payload/AuditRedactionPrimitives.cs @@ -0,0 +1,310 @@ +using System.Text; +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; +using Microsoft.Extensions.Logging; + +namespace ZB.MOM.WW.ScadaBridge.AuditLog.Payload; + +/// +/// Pure, stateless redaction + truncation primitives shared by the legacy +/// (which operates on the +/// entity) +/// and the canonical (which operates on +/// ZB.MOM.WW.Audit.AuditEvent + its DetailsJson). Extracted in +/// ScadaBridge audit re-architecture stage C2 (Task 2.5) so the byte-exact +/// redaction logic lives in ONE place and the two code paths can never drift. +/// +/// +/// +/// Each stage method is a pure function of its inputs (no instance state). The +/// only side effects are diagnostics-only: a warning log line and an +/// callback invocation when a redactor faults, so +/// the caller can bump its redaction-failure health counter. The callbacks are +/// passed in (rather than the counter interface) to keep this helper free of +/// any DI / health-metric coupling. +/// +/// +/// The regex CACHE and per-call options resolution deliberately stay in +/// — they carry per-instance state +/// (lazy compile, 100 ms compile budget, sentinel entries) that the safety-net +/// tests pin to that class. This helper only holds the stateless stages that +/// operate once the compiled regex set / redact list / cap has already been +/// resolved. +/// +/// +internal static class AuditRedactionPrimitives +{ + /// Marker replacing redacted header values, body matches, and SQL parameter values. + public const string RedactedMarker = ""; + + /// Over-redaction marker emitted when a redactor stage itself faults. + public const string RedactorErrorMarker = ""; + + /// + /// JSON serializer options used to re-emit redacted summaries. The + /// UnsafeRelaxedJsonEscaping encoder is required so the redaction marker + /// (which contains < / >) survives unescaped — matching + /// the legacy filter's output byte-for-byte. + /// + public static readonly JsonSerializerOptions RedactedSummaryJsonOptions = new() + { + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }; + + /// + /// Parse as the documented + /// {"headers": {...}, "body": ...} shape and replace values whose + /// header NAME (case-insensitive) is in with + /// . Re-serialises and returns the result. + /// No-op pass-through for inputs that are not JSON-object-shaped or do not + /// carry a top-level headers object. On any unexpected fault the + /// field is over-redacted with and + /// is invoked. + /// + public static string? RedactHeaders( + string? json, + IList redactList, + ILogger logger, + Action onFailure) + { + if (json is null) + { + return null; + } + + // Cheap structural pre-check: only attempt JSON parsing when the input + // actually looks like a JSON object. Saves the JsonDocument allocation + // on the (very common) non-JSON ErrorDetail / Extra fields. + var trimmed = json.AsSpan().TrimStart(); + if (trimmed.Length == 0 || trimmed[0] != '{') + { + return json; + } + + try + { + JsonNode? root; + try + { + root = JsonNode.Parse(json); + } + catch (JsonException) + { + // Not parseable JSON — leave the field alone (no error, no + // redaction). Emitters not yet using the documented shape get + // a transparent pass. + return json; + } + + if (root is not JsonObject obj || obj["headers"] is not JsonObject headers) + { + // No "headers" object at the top level — nothing to redact. + return json; + } + + // Build a case-insensitive lookup of the redact list so we can do + // one O(1) check per header name without an inner Any() loop. + var redactSet = new HashSet(redactList, StringComparer.OrdinalIgnoreCase); + + // Take a snapshot of names first — we cannot mutate while + // enumerating the JsonObject. + var names = new List(headers.Count); + foreach (var kvp in headers) + { + names.Add(kvp.Key); + } + foreach (var name in names) + { + if (redactSet.Contains(name)) + { + headers[name] = JsonValue.Create(RedactedMarker); + } + } + + return obj.ToJsonString(RedactedSummaryJsonOptions); + } + catch (Exception ex) + { + logger.LogWarning( + ex, + "Header redactor faulted; over-redacting field with '{Marker}'", + RedactorErrorMarker); + try { onFailure(); } catch { /* swallow per §7 */ } + return RedactorErrorMarker; + } + } + + /// + /// Apply each compiled body-redactor regex to in + /// turn, replacing every match with . If any + /// single regex match throws (most commonly + /// ) the field is over-redacted + /// with and + /// is invoked — the user-facing action is never aborted. + /// + public static string? RedactBody( + string? value, + IReadOnlyList regexes, + ILogger logger, + Action onFailure) + { + if (value is null) + { + return null; + } + var current = value; + foreach (var rx in regexes) + { + try + { + current = rx.Replace(current, RedactedMarker); + } + catch (Exception ex) + { + logger.LogWarning( + ex, + "Body redactor '{Pattern}' faulted; over-redacting field with '{Marker}'", + rx.ToString(), RedactorErrorMarker); + try { onFailure(); } catch { /* swallow per §7 */ } + return RedactorErrorMarker; + } + } + return current; + } + + /// + /// Walk the M4 {"sql":"...","parameters":{...}} RequestSummary + /// shape; for each parameter whose NAME matches + /// , replace its value with + /// . Re-serialise. No-op pass-through when the + /// input is not parseable JSON, is not a JSON object, or does not carry a + /// top-level "parameters" object. On any unexpected fault the field + /// is over-redacted with and + /// is invoked. + /// + public static string? RedactSqlParameters( + string? json, + Regex paramNameRegex, + ILogger logger, + Action onFailure) + { + if (json is null) + { + return null; + } + + var trimmed = json.AsSpan().TrimStart(); + if (trimmed.Length == 0 || trimmed[0] != '{') + { + return json; + } + + try + { + JsonNode? root; + try + { + root = JsonNode.Parse(json); + } + catch (JsonException) + { + return json; + } + + if (root is not JsonObject obj || obj["parameters"] is not JsonObject parameters) + { + return json; + } + + // Snapshot the names — mutating during enumeration is unsupported. + var names = new List(parameters.Count); + foreach (var kvp in parameters) + { + names.Add(kvp.Key); + } + var anyChanged = false; + foreach (var name in names) + { + bool matched; + try + { + matched = paramNameRegex.IsMatch(name); + } + catch (Exception ex) + { + logger.LogWarning( + ex, + "SQL parameter redactor faulted; over-redacting field with '{Marker}'", + RedactorErrorMarker); + try { onFailure(); } catch { /* swallow per §7 */ } + return RedactorErrorMarker; + } + if (matched) + { + parameters[name] = JsonValue.Create(RedactedMarker); + anyChanged = true; + } + } + + // Avoid re-serialising (which would normalise whitespace / order) + // when no parameter matched — keeps the on-disk row byte-identical + // to the emitter's output on the no-match path. + return anyChanged ? obj.ToJsonString(RedactedSummaryJsonOptions) : json; + } + catch (Exception ex) + { + logger.LogWarning( + ex, + "SQL parameter redactor faulted; over-redacting field with '{Marker}'", + RedactorErrorMarker); + try { onFailure(); } catch { /* swallow per §7 */ } + return RedactorErrorMarker; + } + } + + /// + /// Truncate to UTF-8 bytes, + /// setting to true when the value was + /// shortened. Null passes through as null. + /// + public static string? TruncateField(string? value, int cap, ref bool truncated) + { + if (value is null) + { + return null; + } + var result = TruncateUtf8(value, cap); + if (result.Length != value.Length) + { + truncated = true; + } + return result; + } + + /// + /// UTF-8 byte-safe truncation. Encodes the input to UTF-8, walks back from + /// the cap position until the byte is NOT a continuation byte + /// (byte & 0xC0 == 0x80), and decodes the resulting prefix — + /// guaranteeing the returned string never splits a multi-byte sequence. + /// + public static string TruncateUtf8(string value, int capBytes) + { + if (string.IsNullOrEmpty(value)) + { + return value; + } + var bytes = Encoding.UTF8.GetBytes(value); + if (bytes.Length <= capBytes) + { + return value; + } + var boundary = capBytes; + while (boundary > 0 && (bytes[boundary] & 0xC0) == 0x80) + { + boundary--; + } + return Encoding.UTF8.GetString(bytes, 0, boundary); + } +} diff --git a/src/ZB.MOM.WW.ScadaBridge.AuditLog/Payload/AuditRegexCache.cs b/src/ZB.MOM.WW.ScadaBridge.AuditLog/Payload/AuditRegexCache.cs new file mode 100644 index 00000000..6d5f0646 --- /dev/null +++ b/src/ZB.MOM.WW.ScadaBridge.AuditLog/Payload/AuditRegexCache.cs @@ -0,0 +1,96 @@ +using System.Collections.Concurrent; +using System.Text.RegularExpressions; +using Microsoft.Extensions.Logging; + +namespace ZB.MOM.WW.ScadaBridge.AuditLog.Payload; + +/// +/// Per-instance compiled-regex cache for audit body / SQL-parameter redactors. +/// Extracted in ScadaBridge audit re-architecture stage C2 (Task 2.5) so the +/// legacy and the canonical +/// +/// share the SAME compile rules (50 ms per-match timeout, 100 ms compile budget, +/// invalid-pattern sentinel) rather than duplicating the logic. +/// +/// +/// +/// Lazy population keyed by pattern string: each pattern is compiled on first +/// use and cached forever. A failed compile (or a compile slower than 100 ms) +/// caches a sentinel so the failing compile is not retried on every event. The +/// failure is logged once on first encounter. +/// is the right primitive because the owning redactor is a DI singleton on the +/// audit hot-path. +/// +/// +internal sealed class AuditRegexCache +{ + /// + /// Per-match regex timeout. Catastrophic-backtracking patterns trip a + /// when a single match takes longer + /// than this; the caller then over-redacts the offending field. 50 ms is + /// generous for normal patterns yet short enough that the audit hot-path is + /// not held up by a misconfigured regex. + /// + private static readonly TimeSpan RegexMatchTimeout = TimeSpan.FromMilliseconds(50); + + private readonly ConcurrentDictionary _cache = new(); + private readonly ILogger _logger; + + public AuditRegexCache(ILogger logger) => _logger = logger; + + /// + /// Resolve a compiled regex from the cache, compiling it on first use. + /// Returns false for patterns that are invalid OR whose compile took + /// longer than 100 ms (the spec calls catastrophic-backtracking guesses at + /// compile time "invalid"); the failure is logged once and the sentinel + /// cache entry prevents repeat compile attempts. + /// + public bool TryGet(string pattern, out Regex? regex) + { + var entry = _cache.GetOrAdd(pattern, Compile); + regex = entry.Regex; + return entry.Regex != null; + } + + private CompiledRegex Compile(string pattern) + { + try + { + var swStart = System.Diagnostics.Stopwatch.GetTimestamp(); + var rx = new Regex(pattern, RegexOptions.Compiled, RegexMatchTimeout); + var elapsedMs = (System.Diagnostics.Stopwatch.GetTimestamp() - swStart) + * 1000d / System.Diagnostics.Stopwatch.Frequency; + if (elapsedMs > 100) + { + _logger.LogWarning( + "Body redactor pattern compiled in {Elapsed}ms (> 100ms cap); rejecting '{Pattern}'", + elapsedMs, pattern); + return CompiledRegex.Invalid; + } + return new CompiledRegex(rx); + } + catch (Exception ex) + { + _logger.LogWarning( + ex, + "Body redactor pattern '{Pattern}' failed to compile; skipping", + pattern); + return CompiledRegex.Invalid; + } + } + + /// + /// Cache entry for a body-redactor pattern. Carries the working + /// on the success path, or the + /// sentinel for patterns that failed to compile (or exceeded the 100 ms + /// compile budget). + /// + private readonly struct CompiledRegex + { + public static readonly CompiledRegex Invalid = new(null); + + public Regex? Regex { get; } + + public CompiledRegex(Regex? regex) => Regex = regex; + } +} diff --git a/src/ZB.MOM.WW.ScadaBridge.AuditLog/Payload/DefaultAuditPayloadFilter.cs b/src/ZB.MOM.WW.ScadaBridge.AuditLog/Payload/DefaultAuditPayloadFilter.cs index e6f1d5d9..5d36730d 100644 --- a/src/ZB.MOM.WW.ScadaBridge.AuditLog/Payload/DefaultAuditPayloadFilter.cs +++ b/src/ZB.MOM.WW.ScadaBridge.AuditLog/Payload/DefaultAuditPayloadFilter.cs @@ -1,8 +1,3 @@ -using System.Collections.Concurrent; -using System.Text; -using System.Text.Encodings.Web; -using System.Text.Json; -using System.Text.Json.Nodes; using System.Text.RegularExpressions; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -55,48 +50,15 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Payload; /// public sealed class DefaultAuditPayloadFilter : IAuditPayloadFilter { - private const string RedactedMarker = ""; - private const string RedactorErrorMarker = ""; - - /// - /// Per-match regex timeout. Catastrophic-backtracking patterns trip a - /// when a single match takes - /// longer than this; the offending field is then over-redacted with - /// and the failure counter is bumped. - /// 50 ms is generous for normal patterns yet short enough that the - /// audit hot-path isn't held up by a misconfigured regex. - /// - private static readonly TimeSpan RegexMatchTimeout = TimeSpan.FromMilliseconds(50); - - /// - /// JSON serializer options used to re-emit redacted summaries. The - /// UnsafeRelaxedJsonEscaping encoder is required so the redaction marker - /// (which contains < / >) survives unescaped — the - /// header-redaction tests grep for the literal marker, and the downstream - /// UI / log readers would rather see <redacted> than - /// . The summaries are persisted to the audit - /// table and rendered in trusted-internal contexts only, so the relaxed - /// HTML-escaping rules do not introduce an XSS surface. - /// - private static readonly JsonSerializerOptions RedactedSummaryJsonOptions = new() - { - Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, - }; - + // Redaction markers + the relaxed-escaping JSON options live in + // AuditRedactionPrimitives, and the compiled-regex cache (50 ms match + // timeout, 100 ms compile budget, invalid-pattern sentinel) lives in + // AuditRegexCache — both shared C2 helpers so the legacy filter and the + // canonical ScadaBridgeAuditRedactor emit byte-identical output. private readonly IOptionsMonitor _options; private readonly ILogger _logger; private readonly IAuditRedactionFailureCounter _failureCounter; - - /// - /// Compiled-regex cache keyed by pattern string. Lazy population: each - /// pattern is compiled on first use and cached forever (the entry's - /// carries either the working - /// or a sentinel marking the pattern as invalid so we don't retry the - /// failing compile on every call). ConcurrentDictionary is the right - /// thread-safety primitive here because the filter is a DI singleton - /// shared across the audit hot-path. - /// - private readonly ConcurrentDictionary _regexCache = new(); + private readonly AuditRegexCache _regexCache; /// /// Primary constructor used by DI — pulls the optional redaction-failure @@ -114,6 +76,7 @@ public sealed class DefaultAuditPayloadFilter : IAuditPayloadFilter _options = options ?? throw new ArgumentNullException(nameof(options)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _failureCounter = failureCounter ?? new NoOpAuditRedactionFailureCounter(); + _regexCache = new AuditRegexCache(_logger); } /// @@ -198,83 +161,18 @@ public sealed class DefaultAuditPayloadFilter : IAuditPayloadFilter /// Parse as the documented /// {"headers": {...}, "body": ...} shape and replace values whose /// header NAME (case-insensitive) is in with - /// . Re-serialises and returns the result. + /// the redaction marker. Re-serialises and returns the result. Delegates to + /// . /// /// /// No-op pass-through for inputs that aren't JSON-shaped — emitters that /// have not yet adopted the convention (the M2 site emitters today, which /// leave RequestSummary null on outbound API calls) get a transparent /// pass. If the redactor itself throws, we over-redact the whole field - /// with and bump the failure counter. + /// with the redactor-error marker and bump the failure counter. /// private string? RedactHeaders(string? json, IList redactList) - { - if (json is null) - { - return null; - } - - // Cheap structural pre-check: only attempt JSON parsing when the input - // actually looks like a JSON object. Saves the JsonDocument allocation - // on the (very common) non-JSON ErrorDetail / Extra fields. - var trimmed = json.AsSpan().TrimStart(); - if (trimmed.Length == 0 || trimmed[0] != '{') - { - return json; - } - - try - { - JsonNode? root; - try - { - root = JsonNode.Parse(json); - } - catch (JsonException) - { - // Not parseable JSON — leave the field alone (no error, no - // redaction). Emitters not yet using the documented shape get - // a transparent pass; Bundle C will update them. - return json; - } - - if (root is not JsonObject obj || obj["headers"] is not JsonObject headers) - { - // No "headers" object at the top level — nothing to redact. - return json; - } - - // Build a case-insensitive lookup of the redact list so we can do - // one O(1) check per header name without an inner Any() loop. - var redactSet = new HashSet(redactList, StringComparer.OrdinalIgnoreCase); - - // Take a snapshot of names first — we cannot mutate while - // enumerating the JsonObject. - var names = new List(headers.Count); - foreach (var kvp in headers) - { - names.Add(kvp.Key); - } - foreach (var name in names) - { - if (redactSet.Contains(name)) - { - headers[name] = JsonValue.Create(RedactedMarker); - } - } - - return obj.ToJsonString(RedactedSummaryJsonOptions); - } - catch (Exception ex) - { - _logger.LogWarning( - ex, - "Header redactor faulted; over-redacting field with '{Marker}'", - RedactorErrorMarker); - try { _failureCounter.Increment(); } catch { /* swallow per §7 */ } - return RedactorErrorMarker; - } - } + => AuditRedactionPrimitives.RedactHeaders(json, redactList, _logger, IncrementFailureCounter); /// /// Combine the global and per-target body-redactor lists for a single @@ -301,7 +199,7 @@ public sealed class DefaultAuditPayloadFilter : IAuditPayloadFilter { foreach (var pattern in opts.GlobalBodyRedactors) { - if (TryGetCompiledRegex(pattern, out var rx)) + if (_regexCache.TryGet(pattern, out var rx)) { result.Add(rx!); } @@ -311,7 +209,7 @@ public sealed class DefaultAuditPayloadFilter : IAuditPayloadFilter { foreach (var pattern in perTargetAdditions) { - if (TryGetCompiledRegex(pattern, out var rx)) + if (_regexCache.TryGet(pattern, out var rx)) { result.Add(rx!); } @@ -320,80 +218,17 @@ public sealed class DefaultAuditPayloadFilter : IAuditPayloadFilter return result; } - /// - /// Resolve a compiled regex from the cache, compiling it on first use. - /// Returns false for patterns that are invalid OR whose compile - /// took longer than 100 ms (the spec calls catastrophic-backtracking - /// guesses at compile time "invalid"); the failure is logged once and - /// the sentinel cache entry prevents repeat compile attempts. - /// - private bool TryGetCompiledRegex(string pattern, out Regex? regex) - { - var entry = _regexCache.GetOrAdd(pattern, CompileRegex); - regex = entry.Regex; - return entry.Regex != null; - } - - private CompiledRegex CompileRegex(string pattern) - { - try - { - var swStart = System.Diagnostics.Stopwatch.GetTimestamp(); - var rx = new Regex(pattern, RegexOptions.Compiled, RegexMatchTimeout); - var elapsedMs = (System.Diagnostics.Stopwatch.GetTimestamp() - swStart) - * 1000d / System.Diagnostics.Stopwatch.Frequency; - if (elapsedMs > 100) - { - _logger.LogWarning( - "Body redactor pattern compiled in {Elapsed}ms (> 100ms cap); rejecting '{Pattern}'", - elapsedMs, pattern); - return CompiledRegex.Invalid; - } - return new CompiledRegex(rx); - } - catch (Exception ex) - { - _logger.LogWarning( - ex, - "Body redactor pattern '{Pattern}' failed to compile; skipping", - pattern); - return CompiledRegex.Invalid; - } - } - /// /// Apply each compiled body-redactor regex to in - /// turn, replacing every match with . If any - /// single regex match throws (most commonly - /// ) the field is over-redacted - /// with and the failure counter is - /// incremented — the user-facing action is never aborted. + /// turn, replacing every match with the redaction marker. If any single + /// regex match throws (most commonly + /// ) the field is over-redacted with + /// the redactor-error marker and the failure counter is incremented — the + /// user-facing action is never aborted. Delegates to + /// . /// private string? RedactBody(string? value, IReadOnlyList regexes) - { - if (value is null) - { - return null; - } - var current = value; - foreach (var rx in regexes) - { - try - { - current = rx.Replace(current, RedactedMarker); - } - catch (Exception ex) - { - _logger.LogWarning( - ex, - "Body redactor '{Pattern}' faulted; over-redacting field with '{Marker}'", - rx.ToString(), RedactorErrorMarker); - try { _failureCounter.Increment(); } catch { /* swallow per §7 */ } - return RedactorErrorMarker; - } - } - return current; - } + => AuditRedactionPrimitives.RedactBody(value, regexes, _logger, IncrementFailureCounter); /// /// Resolve the per-connection SQL parameter redaction regex for the given @@ -425,7 +260,7 @@ public sealed class DefaultAuditPayloadFilter : IAuditPayloadFilter // the pattern without an IgnoreCase flag. The compile cache key folds // the option to keep the entries unambiguous. var cacheKey = "(?i)" + over.RedactSqlParamsMatching; - if (!TryGetCompiledRegex(cacheKey, out regex)) + if (!_regexCache.TryGet(cacheKey, out regex)) { return false; } @@ -435,128 +270,30 @@ public sealed class DefaultAuditPayloadFilter : IAuditPayloadFilter /// /// Walk the M4 {"sql":"...","parameters":{...}} RequestSummary /// shape; for each parameter whose NAME matches - /// , replace its value with - /// . Re-serialise. + /// , replace its value with the redaction + /// marker. Re-serialise. Delegates to + /// . /// /// /// No-op pass-through when the input isn't parseable JSON, isn't a JSON /// object, or doesn't carry a top-level "parameters" object. On - /// any unexpected fault the field is over-redacted with - /// and the failure counter is bumped. + /// any unexpected fault the field is over-redacted and the failure counter + /// is bumped. /// private string? RedactSqlParameters(string? json, Regex paramNameRegex) - { - if (json is null) - { - return null; - } - - var trimmed = json.AsSpan().TrimStart(); - if (trimmed.Length == 0 || trimmed[0] != '{') - { - return json; - } - - try - { - JsonNode? root; - try - { - root = JsonNode.Parse(json); - } - catch (JsonException) - { - return json; - } - - if (root is not JsonObject obj || obj["parameters"] is not JsonObject parameters) - { - return json; - } - - // Snapshot the names — mutating during enumeration is unsupported. - var names = new List(parameters.Count); - foreach (var kvp in parameters) - { - names.Add(kvp.Key); - } - var anyChanged = false; - foreach (var name in names) - { - bool matched; - try - { - matched = paramNameRegex.IsMatch(name); - } - catch (Exception ex) - { - _logger.LogWarning( - ex, - "SQL parameter redactor faulted; over-redacting field with '{Marker}'", - RedactorErrorMarker); - try { _failureCounter.Increment(); } catch { /* swallow per §7 */ } - return RedactorErrorMarker; - } - if (matched) - { - parameters[name] = JsonValue.Create(RedactedMarker); - anyChanged = true; - } - } - - // Avoid re-serialising (which would normalise whitespace / order) - // when no parameter matched — keeps the on-disk row byte-identical - // to the emitter's output on the no-match path. - return anyChanged ? obj.ToJsonString(RedactedSummaryJsonOptions) : json; - } - catch (Exception ex) - { - _logger.LogWarning( - ex, - "SQL parameter redactor faulted; over-redacting field with '{Marker}'", - RedactorErrorMarker); - try { _failureCounter.Increment(); } catch { /* swallow per §7 */ } - return RedactorErrorMarker; - } - } + => AuditRedactionPrimitives.RedactSqlParameters(json, paramNameRegex, _logger, IncrementFailureCounter); private static string? TruncateField(string? value, int cap, ref bool truncated) - { - if (value is null) - { - return null; - } - var result = TruncateUtf8(value, cap); - if (result.Length != value.Length) - { - truncated = true; - } - return result; - } + => AuditRedactionPrimitives.TruncateField(value, cap, ref truncated); /// - /// UTF-8 byte-safe truncation. Encodes the input to UTF-8, walks back from - /// the cap position until the byte is NOT a continuation byte - /// (byte & 0xC0 == 0x80), and decodes the resulting prefix — - /// guaranteeing the returned string never splits a multi-byte sequence. + /// Bumps the injected redaction-failure counter, swallowing any fault per + /// alog.md §7 (a counter failure must never abort the audited action). + /// Passed as the onFailure callback to the shared primitives. /// - private static string TruncateUtf8(string value, int capBytes) + private void IncrementFailureCounter() { - if (string.IsNullOrEmpty(value)) - { - return value; - } - var bytes = Encoding.UTF8.GetBytes(value); - if (bytes.Length <= capBytes) - { - return value; - } - var boundary = capBytes; - while (boundary > 0 && (bytes[boundary] & 0xC0) == 0x80) - { - boundary--; - } - return Encoding.UTF8.GetString(bytes, 0, boundary); + try { _failureCounter.Increment(); } catch { /* swallow per §7 */ } } private static bool IsErrorStatus(AuditStatus status) => status switch @@ -564,24 +301,4 @@ public sealed class DefaultAuditPayloadFilter : IAuditPayloadFilter AuditStatus.Delivered or AuditStatus.Submitted or AuditStatus.Forwarded => false, _ => true, }; - - /// - /// Cache entry for a body-redactor pattern. Carries the working - /// on the success path, or the - /// sentinel for patterns that failed to compile - /// (or exceeded the 100 ms compile budget). The sentinel lets us skip - /// repeat compile attempts on every event without re-throwing on the - /// hot-path. - /// - private readonly struct CompiledRegex - { - public static readonly CompiledRegex Invalid = new(null); - - /// Gets the compiled , or null when the pattern was invalid. - public Regex? Regex { get; } - - /// Initializes a new wrapping the given compiled regex instance. - /// The pre-compiled regex, or null to represent an invalid pattern. - public CompiledRegex(Regex? regex) => Regex = regex; - } } diff --git a/src/ZB.MOM.WW.ScadaBridge.AuditLog/Redaction/SafeDefaultAuditRedactor.cs b/src/ZB.MOM.WW.ScadaBridge.AuditLog/Redaction/SafeDefaultAuditRedactor.cs new file mode 100644 index 00000000..29940916 --- /dev/null +++ b/src/ZB.MOM.WW.ScadaBridge.AuditLog/Redaction/SafeDefaultAuditRedactor.cs @@ -0,0 +1,91 @@ +using System.Text.RegularExpressions; +using ZB.MOM.WW.Audit; +using ZB.MOM.WW.ScadaBridge.AuditLog.Payload; +using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; + +namespace ZB.MOM.WW.ScadaBridge.AuditLog.Redaction; + +/// +/// Canonical-record analogue of for +/// stage C2 (Task 2.5): a minimal always-safe +/// fallback for composition roots that bypass the full +/// . Performs line-oriented HTTP header +/// redaction for the always-sensitive defaults (Authorization, X-Api-Key, +/// Cookie, Set-Cookie) on the RequestSummary / ResponseSummary +/// fields carried inside ZB.MOM.WW.Audit.AuditEvent.DetailsJson. Does NOT +/// perform body-regex redaction, SQL-parameter redaction, or truncation — those +/// need with live options. Contract: +/// over-redact safely, never throw, never miss a header on the default +/// sensitive list. +/// +public sealed class SafeDefaultAuditRedactor : IAuditRedactor +{ + /// Singleton instance — the redactor is stateless and side-effect-free. + public static SafeDefaultAuditRedactor Instance { get; } = new SafeDefaultAuditRedactor(); + + private static readonly string[] DefaultHeaderRedactList = + { + "Authorization", + "X-Api-Key", + "Cookie", + "Set-Cookie", + }; + + private static readonly Regex HeaderRegex = new( + @"(?[A-Za-z][A-Za-z0-9\-_]*)\s*:\s*(?[^\r\n]*)", + RegexOptions.Compiled | RegexOptions.IgnoreCase); + + private SafeDefaultAuditRedactor() { } + + /// + public AuditEvent Apply(AuditEvent rawEvent) + { + ArgumentNullException.ThrowIfNull(rawEvent); + + // Fast path: no DetailsJson means no summaries to scrub. + if (string.IsNullOrEmpty(rawEvent.DetailsJson)) + { + return rawEvent; + } + + try + { + var d = AuditDetailsCodec.Deserialize(rawEvent.DetailsJson); + var scrubbed = d with + { + RequestSummary = RedactHeaders(d.RequestSummary), + ResponseSummary = RedactHeaders(d.ResponseSummary), + }; + return rawEvent with { DetailsJson = AuditDetailsCodec.Serialize(scrubbed) }; + } + catch + { + // Over-redact: drop both summaries entirely so a malformed parse + // path never leaks the original. The contract is "never throw." + var safe = new AuditDetails + { + RequestSummary = "[redacted by SafeDefaultAuditRedactor]", + ResponseSummary = "[redacted by SafeDefaultAuditRedactor]", + }; + return rawEvent with { DetailsJson = AuditDetailsCodec.Serialize(safe) }; + } + } + + private static string? RedactHeaders(string? summary) + { + if (string.IsNullOrEmpty(summary)) return summary; + + return HeaderRegex.Replace(summary, m => + { + var name = m.Groups["name"].Value; + foreach (var sensitive in DefaultHeaderRedactList) + { + if (string.Equals(name, sensitive, StringComparison.OrdinalIgnoreCase)) + { + return $"{name}: [REDACTED]"; + } + } + return m.Value; + }); + } +} diff --git a/src/ZB.MOM.WW.ScadaBridge.AuditLog/Redaction/ScadaBridgeAuditRedactor.cs b/src/ZB.MOM.WW.ScadaBridge.AuditLog/Redaction/ScadaBridgeAuditRedactor.cs new file mode 100644 index 00000000..2db12e7b --- /dev/null +++ b/src/ZB.MOM.WW.ScadaBridge.AuditLog/Redaction/ScadaBridgeAuditRedactor.cs @@ -0,0 +1,349 @@ +using System.Text; +using System.Text.RegularExpressions; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using ZB.MOM.WW.Audit; +using ZB.MOM.WW.ScadaBridge.AuditLog.Configuration; +using ZB.MOM.WW.ScadaBridge.AuditLog.Payload; +using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; +using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; + +namespace ZB.MOM.WW.ScadaBridge.AuditLog.Redaction; + +/// +/// Canonical implementation for ScadaBridge — the +/// stage-C2 port of onto +/// ZB.MOM.WW.Audit.AuditEvent and its +/// payload bag. The ScadaBridge request/response/error/extra summaries travel +/// inside DetailsJson as a record (serialized +/// by ); this redactor deserializes them, applies +/// the SAME header → body-regex → SQL-parameter → byte-safe truncation pipeline +/// the legacy filter applies, re-serializes, and returns a filtered COPY. +/// +/// +/// +/// Additive only: the legacy pipeline stays in +/// place and wired until stage C3 swaps the record type; this redactor is the +/// canonical-record analogue exercised in isolation by the C2 unit tests. +/// +/// +/// Cap selection is faithful to the legacy filter, translated onto canonical +/// fields: +/// +/// The ApiInbound branch keys on +/// (= AuditChannel.ToString() per ) +/// → . +/// The "error row" branch reproduces the legacy +/// IsErrorStatus(Status) rule — Status NOT IN (Delivered, +/// Submitted, Forwarded) → . +/// The fine-grained status is read from +/// when present (it must be — alone cannot +/// reproduce IsErrorStatus, since Attempted/Skipped +/// project to yet take the error cap). +/// When is absent/unparseable the +/// canonical is the fallback: +/// / +/// → error cap. +/// +/// +/// +/// MUST NOT throw — wrapped in try/catch; over-redacts (drops the summaries to a +/// safe marker) on any internal failure, mirroring +/// . +/// +/// +public sealed class ScadaBridgeAuditRedactor : IAuditRedactor +{ + private const string OverRedactedMarker = "[redacted by ScadaBridgeAuditRedactor]"; + + private readonly IOptionsMonitor _options; + private readonly ILogger _logger; + private readonly IAuditRedactionFailureCounter _failureCounter; + private readonly AuditRegexCache _regexCache; + + /// + /// Primary constructor used by DI — pulls the optional redaction-failure + /// counter from the container; a NoOp default is used when none is supplied. + /// + /// Live-reloadable audit log options. + /// Logger for redaction diagnostics. + /// Optional counter incremented when a redaction operation fails; defaults to a no-op. + public ScadaBridgeAuditRedactor( + IOptionsMonitor options, + ILogger logger, + IAuditRedactionFailureCounter? failureCounter = null) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _failureCounter = failureCounter ?? new NoOpAuditRedactionFailureCounter(); + _regexCache = new AuditRegexCache(_logger); + } + + /// + public AuditEvent Apply(AuditEvent rawEvent) + { + try + { + var opts = _options.CurrentValue; + + // --- Fast path ------------------------------------------------- + // Mirror the legacy filter's non-JSON pre-check: when there is no + // DetailsJson payload to scrub AND the Target is within the cap, + // there is nothing to redact or truncate. Return the input + // unchanged so the common case stays cheap (no Deserialize, no + // re-Serialize, same instance back). + var detailsEmpty = string.IsNullOrEmpty(rawEvent.DetailsJson); + var targetWithinCap = rawEvent.Target is null + || Encoding.UTF8.GetByteCount(rawEvent.Target) <= opts.DefaultCapBytes; + if (detailsEmpty && targetWithinCap) + { + return rawEvent; + } + + // --- Slow path ------------------------------------------------- + var d = AuditDetailsCodec.Deserialize(rawEvent.DetailsJson); + + // Cap selection. Channel = canonical Category (the ApiInbound + // branch); error-cap selection reproduces the legacy + // IsErrorStatus(Status) — read from d.Status when present, else + // fall back to the canonical Outcome. + var cap = SelectCap(opts, rawEvent.Category, d.Status, rawEvent.Outcome); + + // --- Header-redaction stage (runs BEFORE truncation) ---------- + var request = RedactHeaders(d.RequestSummary, opts.HeaderRedactList); + var response = RedactHeaders(d.ResponseSummary, opts.HeaderRedactList); + var errorDetail = d.ErrorDetail; + var extra = d.Extra; + + // --- Body-regex stage (also runs BEFORE truncation) ----------- + // Per-target additions key on the canonical Target. + var bodyRegexes = ResolveBodyRegexes(opts, rawEvent.Target); + if (bodyRegexes.Count > 0) + { + request = RedactBody(request, bodyRegexes); + response = RedactBody(response, bodyRegexes); + errorDetail = RedactBody(errorDetail, bodyRegexes); + extra = RedactBody(extra, bodyRegexes); + } + + // --- SQL parameter redaction stage (DbOutbound only) ---------- + // Channel-guarded on the canonical Category; connection key is the + // Target prefix before the first '.'. + if (string.Equals(rawEvent.Category, nameof(AuditChannel.DbOutbound), StringComparison.Ordinal) + && TryGetSqlParamRedactor(opts, rawEvent.Target, out var sqlParamRegex)) + { + request = RedactSqlParameters(request, sqlParamRegex!); + } + + // --- Truncation stage ----------------------------------------- + var truncated = false; + request = TruncateField(request, cap, ref truncated); + response = TruncateField(response, cap, ref truncated); + errorDetail = TruncateField(errorDetail, cap, ref truncated); + extra = TruncateField(extra, cap, ref truncated); + + var rewritten = d with + { + RequestSummary = request, + ResponseSummary = response, + ErrorDetail = errorDetail, + Extra = extra, + PayloadTruncated = d.PayloadTruncated || truncated, + }; + + // Target length cap (canonical top-level field). Cap at the default + // byte ceiling so an absurd Target cannot blow the storage column. + var cappedTarget = TruncateTarget(rawEvent.Target, opts.DefaultCapBytes); + + return rawEvent with + { + DetailsJson = AuditDetailsCodec.Serialize(rewritten), + Target = cappedTarget, + }; + } + catch (Exception ex) + { + // Audit is best-effort: over-redact rather than fail the caller. + // Drop the summaries entirely (mirroring SafeDefault's catch path) + // and flag PayloadTruncated so downstream readers know the row was + // scrubbed defensively. + _logger.LogWarning( + ex, + "Canonical audit redactor failed; over-redacting DetailsJson and flagging PayloadTruncated"); + IncrementFailureCounter(); + return OverRedact(rawEvent); + } + } + + /// + /// Pick the truncation cap. = canonical Category + /// (= channel name): ApiInbound → . + /// Otherwise the legacy IsErrorStatus rule decides between the error + /// and default caps, preferring the fine-grained + /// (from DetailsJson) and falling back to the canonical + /// when status is absent/unparseable. + /// + private static int SelectCap( + AuditLogOptions opts, + string? category, + string? detailsStatus, + AuditOutcome outcome) + { + if (string.Equals(category, nameof(AuditChannel.ApiInbound), StringComparison.Ordinal)) + { + return opts.InboundMaxBytes; + } + return IsErrorRow(detailsStatus, outcome) ? opts.ErrorCapBytes : opts.DefaultCapBytes; + } + + /// + /// Reproduce the legacy IsErrorStatus(Status) error-cap predicate on + /// the canonical record: Status NOT IN (Delivered, Submitted, + /// Forwarded) → error row. When the fine-grained status is present in + /// DetailsJson it is authoritative; otherwise the canonical + /// is the fallback + /// (/ + /// → error row). + /// + private static bool IsErrorRow(string? detailsStatus, AuditOutcome outcome) + { + if (!string.IsNullOrEmpty(detailsStatus) + && Enum.TryParse(detailsStatus, ignoreCase: false, out var status)) + { + return status switch + { + AuditStatus.Delivered or AuditStatus.Submitted or AuditStatus.Forwarded => false, + _ => true, + }; + } + // No usable status — fall back to the canonical outcome. + return outcome != AuditOutcome.Success; + } + + private string? RedactHeaders(string? json, IList redactList) + => AuditRedactionPrimitives.RedactHeaders(json, redactList, _logger, IncrementFailureCounter); + + private string? RedactBody(string? value, IReadOnlyList regexes) + => AuditRedactionPrimitives.RedactBody(value, regexes, _logger, IncrementFailureCounter); + + private string? RedactSqlParameters(string? json, Regex paramNameRegex) + => AuditRedactionPrimitives.RedactSqlParameters(json, paramNameRegex, _logger, IncrementFailureCounter); + + private static string? TruncateField(string? value, int cap, ref bool truncated) + => AuditRedactionPrimitives.TruncateField(value, cap, ref truncated); + + private static string? TruncateTarget(string? target, int cap) + => target is null ? null : AuditRedactionPrimitives.TruncateUtf8(target, cap); + + /// + /// Combine the global and per-target body-redactor lists, returning the + /// compiled-regex set to apply. Patterns that failed compilation are + /// silently skipped. Identical resolution to + /// . + /// + private IReadOnlyList ResolveBodyRegexes(AuditLogOptions opts, string? target) + { + var hasGlobal = opts.GlobalBodyRedactors is { Count: > 0 }; + var perTargetAdditions = (target != null + && opts.PerTargetOverrides.TryGetValue(target, out var over) + && over.AdditionalBodyRedactors is { Count: > 0 }) + ? over.AdditionalBodyRedactors + : null; + + if (!hasGlobal && perTargetAdditions == null) + { + return Array.Empty(); + } + + var result = new List(); + if (hasGlobal) + { + foreach (var pattern in opts.GlobalBodyRedactors) + { + if (_regexCache.TryGet(pattern, out var rx)) + { + result.Add(rx!); + } + } + } + if (perTargetAdditions != null) + { + foreach (var pattern in perTargetAdditions) + { + if (_regexCache.TryGet(pattern, out var rx)) + { + result.Add(rx!); + } + } + } + return result; + } + + /// + /// Resolve the per-connection SQL parameter redaction regex for the given + /// target. Connection key = everything before the first . in + /// . Patterns are forced case-insensitive. + /// Identical resolution to . + /// + private bool TryGetSqlParamRedactor(AuditLogOptions opts, string? target, out Regex? regex) + { + regex = null; + if (string.IsNullOrEmpty(target)) + { + return false; + } + + var dot = target.IndexOf('.'); + var connectionKey = dot < 0 ? target : target[..dot]; + + if (!opts.PerTargetOverrides.TryGetValue(connectionKey, out var over) + || string.IsNullOrEmpty(over.RedactSqlParamsMatching)) + { + return false; + } + + var cacheKey = "(?i)" + over.RedactSqlParamsMatching; + return _regexCache.TryGet(cacheKey, out regex); + } + + /// + /// Over-redaction copy returned from the never-throws catch: drop the + /// request/response summaries inside DetailsJson to a safe marker and + /// flag . Best-effort re-serialise; + /// if even that fails, return the input with no summaries via an empty + /// details bag. + /// + private static AuditEvent OverRedact(AuditEvent rawEvent) + { + try + { + var d = AuditDetailsCodec.Deserialize(rawEvent.DetailsJson) with + { + RequestSummary = OverRedactedMarker, + ResponseSummary = OverRedactedMarker, + PayloadTruncated = true, + }; + return rawEvent with { DetailsJson = AuditDetailsCodec.Serialize(d) }; + } + catch + { + var safe = new AuditDetails + { + RequestSummary = OverRedactedMarker, + ResponseSummary = OverRedactedMarker, + PayloadTruncated = true, + }; + return rawEvent with { DetailsJson = AuditDetailsCodec.Serialize(safe) }; + } + } + + /// + /// Bumps the injected redaction-failure counter, swallowing any fault per + /// alog.md §7. Passed as the onFailure callback to the shared + /// primitives and called from the top-level catch. + /// + private void IncrementFailureCounter() + { + try { _failureCounter.Increment(); } catch { /* swallow per §7 */ } + } +} diff --git a/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Redaction/SafeDefaultAuditRedactorTests.cs b/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Redaction/SafeDefaultAuditRedactorTests.cs new file mode 100644 index 00000000..cb74fc8b --- /dev/null +++ b/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Redaction/SafeDefaultAuditRedactorTests.cs @@ -0,0 +1,122 @@ +using ZB.MOM.WW.Audit; +using ZB.MOM.WW.ScadaBridge.AuditLog.Redaction; +using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; + +namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Redaction; + +/// +/// ScadaBridge audit re-architecture stage C2 (Task 2.5) tests for +/// — the canonical-record analogue of +/// . +/// Header-only scrub of the always-sensitive default headers inside +/// DetailsJson's RequestSummary / ResponseSummary; never throws, never +/// performs body / SQL / truncation work. +/// +public class SafeDefaultAuditRedactorTests +{ + private static AuditEvent EventWith(string? request = null, string? response = null) + { + var details = new AuditDetails + { + RequestSummary = request, + ResponseSummary = response, + }; + return new AuditEvent + { + EventId = Guid.NewGuid(), + OccurredAtUtc = DateTimeOffset.UtcNow, + Actor = "tester", + Action = "ApiOutbound.ApiCall", + Outcome = AuditOutcome.Success, + DetailsJson = AuditDetailsCodec.Serialize(details), + }; + } + + private static AuditDetails Details(AuditEvent evt) => + AuditDetailsCodec.Deserialize(evt.DetailsJson); + + [Fact] + public void Redacts_DefaultSensitiveHeaders_InRequestSummary() + { + var evt = EventWith(request: "Authorization: Bearer secret-token\nContent-Type: application/json"); + + var result = SafeDefaultAuditRedactor.Instance.Apply(evt); + + var d = Details(result); + Assert.Contains("Authorization: [REDACTED]", d.RequestSummary!); + Assert.DoesNotContain("secret-token", d.RequestSummary!); + Assert.Contains("Content-Type: application/json", d.RequestSummary!); + } + + [Fact] + public void Redacts_DefaultSensitiveHeaders_InResponseSummary() + { + var evt = EventWith(response: "Set-Cookie: sessionid=abc123\nX-Other: ok"); + + var result = SafeDefaultAuditRedactor.Instance.Apply(evt); + + var d = Details(result); + Assert.Contains("Set-Cookie: [REDACTED]", d.ResponseSummary!); + Assert.DoesNotContain("abc123", d.ResponseSummary!); + Assert.Contains("X-Other: ok", d.ResponseSummary!); + } + + [Fact] + public void CaseInsensitive_HeaderName_Redacted() + { + var evt = EventWith(request: "authorization: Bearer x-y-z"); + + var result = SafeDefaultAuditRedactor.Instance.Apply(evt); + + Assert.Contains("[REDACTED]", Details(result).RequestSummary!); + Assert.DoesNotContain("x-y-z", Details(result).RequestSummary!); + } + + [Fact] + public void NonSensitiveHeader_Preserved() + { + var evt = EventWith(request: "X-Request-Id: abc-123\nAccept: application/json"); + + var result = SafeDefaultAuditRedactor.Instance.Apply(evt); + + var d = Details(result); + Assert.Contains("X-Request-Id: abc-123", d.RequestSummary!); + Assert.Contains("Accept: application/json", d.RequestSummary!); + } + + [Fact] + public void NullDetails_FastPath_ReturnsSameInstance() + { + var evt = new AuditEvent + { + EventId = Guid.NewGuid(), + OccurredAtUtc = DateTimeOffset.UtcNow, + Actor = "tester", + Action = "ApiOutbound.ApiCall", + Outcome = AuditOutcome.Success, + DetailsJson = null, + }; + + var result = SafeDefaultAuditRedactor.Instance.Apply(evt); + + Assert.Same(evt, result); + } + + [Fact] + public void MalformedDetailsJson_NeverThrows() + { + var evt = new AuditEvent + { + EventId = Guid.NewGuid(), + OccurredAtUtc = DateTimeOffset.UtcNow, + Actor = "tester", + Action = "ApiOutbound.ApiCall", + Outcome = AuditOutcome.Success, + DetailsJson = "{not valid json]", + }; + + var ex = Record.Exception(() => SafeDefaultAuditRedactor.Instance.Apply(evt)); + + Assert.Null(ex); + } +} diff --git a/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Redaction/ScadaBridgeAuditRedactorTests.cs b/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Redaction/ScadaBridgeAuditRedactorTests.cs new file mode 100644 index 00000000..ce271f7f --- /dev/null +++ b/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Redaction/ScadaBridgeAuditRedactorTests.cs @@ -0,0 +1,540 @@ +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using ZB.MOM.WW.Audit; +using ZB.MOM.WW.ScadaBridge.AuditLog.Configuration; +using ZB.MOM.WW.ScadaBridge.AuditLog.Payload; +using ZB.MOM.WW.ScadaBridge.AuditLog.Redaction; +using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; +using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; + +namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Redaction; + +/// +/// ScadaBridge audit re-architecture stage C2 (Task 2.5) tests for +/// — the canonical +/// implementation that ports the +/// redaction + truncation behaviour onto +/// the canonical ZB.MOM.WW.Audit.AuditEvent record and its +/// payload bag. These mirror the legacy +/// Payload fixtures (HeaderRedaction / BodyRegex / SqlParam / RedactionSafetyNet +/// / Truncation) but operate on canonical events built via +/// . +/// +public class ScadaBridgeAuditRedactorTests +{ + private static ScadaBridgeAuditRedactor Redactor( + AuditLogOptions? opts = null, + IAuditRedactionFailureCounter? counter = null) => + new(new StaticMonitor(opts ?? new AuditLogOptions()), + NullLogger.Instance, + counter); + + /// + /// Build a canonical event whose carries the + /// supplied summaries + channel/status, mirroring what the C3 emit boundary + /// will produce. Category = channel name (so the ApiInbound branch is keyed + /// correctly); Status travels inside DetailsJson for the fine-grained error + /// cap selection. + /// + private static AuditEvent NewEvent( + AuditChannel channel = AuditChannel.ApiOutbound, + AuditStatus status = AuditStatus.Delivered, + AuditOutcome outcome = AuditOutcome.Success, + string? request = null, + string? response = null, + string? errorDetail = null, + string? extra = null, + string? target = null, + bool detailsPayloadTruncated = false) + { + var details = new AuditDetails + { + Channel = channel.ToString(), + Status = status.ToString(), + RequestSummary = request, + ResponseSummary = response, + ErrorDetail = errorDetail, + Extra = extra, + PayloadTruncated = detailsPayloadTruncated, + }; + return new AuditEvent + { + EventId = Guid.NewGuid(), + OccurredAtUtc = DateTimeOffset.UtcNow, + Actor = "tester", + Action = AuditFieldBuilders.BuildAction(channel, AuditKind.ApiCall), + Category = AuditFieldBuilders.BuildCategory(channel), + Outcome = outcome, + Target = target, + DetailsJson = AuditDetailsCodec.Serialize(details), + }; + } + + private static AuditDetails Details(AuditEvent evt) => + AuditDetailsCodec.Deserialize(evt.DetailsJson); + + // ---- Header redaction (ports HeaderRedactionTests) --------------------- + + [Fact] + public void HeaderRedaction_AuthorizationBearer_Redacted() + { + var request = "{\"headers\":{\"Authorization\":\"Bearer secret-token-xyz\",\"Content-Type\":\"application/json\"},\"body\":\"hello\"}"; + var evt = NewEvent(request: request); + + var result = Redactor().Apply(evt); + + var d = Details(result); + Assert.NotNull(d.RequestSummary); + Assert.Contains("\"Authorization\":\"\"", d.RequestSummary); + Assert.DoesNotContain("secret-token-xyz", d.RequestSummary); + Assert.Contains("application/json", d.RequestSummary); + } + + [Fact] + public void HeaderRedaction_CaseInsensitive_LowercaseAuthorization_Redacted() + { + var request = "{\"headers\":{\"authorization\":\"Bearer secret-token-xyz\"},\"body\":\"hi\"}"; + var evt = NewEvent(request: request); + + var result = Redactor().Apply(evt); + + Assert.Contains("\"authorization\":\"\"", Details(result).RequestSummary!); + } + + [Fact] + public void HeaderRedaction_CustomRedactList_RedactsCustomHeaderName() + { + var opts = new AuditLogOptions { HeaderRedactList = new List { "X-Custom-Secret" } }; + var request = "{\"headers\":{\"X-Custom-Secret\":\"topsecret\",\"Authorization\":\"Bearer keep-me\"},\"body\":\"hi\"}"; + var evt = NewEvent(request: request); + + var result = Redactor(opts).Apply(evt); + + var d = Details(result); + Assert.Contains("\"X-Custom-Secret\":\"\"", d.RequestSummary!); + Assert.Contains("Bearer keep-me", d.RequestSummary!); + } + + [Fact] + public void HeaderRedaction_NonJson_RequestSummary_Unchanged() + { + var evt = NewEvent(request: "this is not JSON at all"); + + var result = Redactor().Apply(evt); + + Assert.Equal("this is not JSON at all", Details(result).RequestSummary); + } + + // ---- Body regex redaction (ports BodyRegexRedactionTests) -------------- + + [Fact] + public void GlobalRegex_HunterPassword_Redacted() + { + var opts = new AuditLogOptions + { + GlobalBodyRedactors = new List { "\"password\":\\s*\"[^\"]*\"" }, + }; + var evt = NewEvent(request: "{\"user\":\"alice\",\"password\":\"hunter2\"}"); + + var result = Redactor(opts).Apply(evt); + + var d = Details(result); + Assert.Contains("", d.RequestSummary!); + Assert.DoesNotContain("hunter2", d.RequestSummary!); + Assert.Contains("alice", d.RequestSummary!); + } + + [Fact] + public void BodyRegex_AppliesToErrorDetailAndExtra() + { + var opts = new AuditLogOptions + { + GlobalBodyRedactors = new List { "SECRET-[A-Z0-9]+" }, + }; + var evt = NewEvent( + errorDetail: "boom SECRET-AAA111 boom", + extra: "ctx SECRET-BBB222 ctx"); + + var result = Redactor(opts).Apply(evt); + + var d = Details(result); + Assert.DoesNotContain("SECRET-AAA111", d.ErrorDetail!); + Assert.DoesNotContain("SECRET-BBB222", d.Extra!); + Assert.Contains("", d.ErrorDetail!); + Assert.Contains("", d.Extra!); + } + + [Fact] + public void PerTargetRegex_OnlyAppliedToMatchingTarget() + { + var opts = new AuditLogOptions + { + PerTargetOverrides = new Dictionary + { + ["esg.A"] = new PerTargetRedactionOverride + { + AdditionalBodyRedactors = new List { "SECRET-[A-Z0-9]+" }, + }, + }, + }; + const string input = "token=SECRET-XYZ123 normal-text"; + + var matched = Redactor(opts).Apply(NewEvent(request: input, target: "esg.A")); + Assert.Contains("", Details(matched).RequestSummary!); + Assert.DoesNotContain("SECRET-XYZ123", Details(matched).RequestSummary!); + + var unmatched = Redactor(opts).Apply(NewEvent(request: input, target: "esg.B")); + Assert.Equal(input, Details(unmatched).RequestSummary); + } + + [Fact] + public void NoRegexConfigured_FieldUnchanged() + { + var evt = NewEvent(request: "{\"password\":\"hunter2\"}"); + + var result = Redactor().Apply(evt); + + Assert.Equal("{\"password\":\"hunter2\"}", Details(result).RequestSummary); + } + + [Fact] + public void RegexThrowsTimeout_FieldBecomesRedactedMarker_CounterIncrements() + { + var opts = new AuditLogOptions { GlobalBodyRedactors = new List { "^(a+)+$" } }; + var counter = new CountingRedactionFailureCounter(); + var evt = NewEvent(request: new string('a', 30) + "!"); + + var result = Redactor(opts, counter).Apply(evt); + + Assert.Equal("", Details(result).RequestSummary); + Assert.True(counter.Count >= 1, $"expected counter >= 1, got {counter.Count}"); + } + + // ---- SQL parameter redaction (ports SqlParamRedactionTests) ------------ + + private static string DbRequestSummary(string sql, params (string name, string value)[] parameters) + { + var sb = new StringBuilder(); + sb.Append("{\"sql\":\"").Append(sql).Append('"'); + if (parameters.Length > 0) + { + sb.Append(",\"parameters\":{"); + for (var i = 0; i < parameters.Length; i++) + { + if (i > 0) sb.Append(','); + sb.Append('"').Append(parameters[i].name).Append("\":\"") + .Append(parameters[i].value).Append('"'); + } + sb.Append('}'); + } + sb.Append('}'); + return sb.ToString(); + } + + [Fact] + public void NoOptIn_ParamsVerbatim_Unchanged() + { + var input = DbRequestSummary( + "INSERT INTO Users (Name, Token) VALUES (@name, @token)", + ("@name", "Alice"), ("@token", "secret-xyz")); + var evt = NewEvent( + channel: AuditChannel.DbOutbound, status: AuditStatus.Delivered, + request: input, target: "PrimaryDb.INSERT INTO Users"); + + var result = Redactor().Apply(evt); + + Assert.Equal(input, Details(result).RequestSummary); + } + + [Fact] + public void OptInRegex_AtToken_RedactsThoseValues_KeepsOthers() + { + var opts = new AuditLogOptions + { + PerTargetOverrides = new Dictionary + { + ["PrimaryDb"] = new PerTargetRedactionOverride { RedactSqlParamsMatching = "^@(token|apikey)$" }, + }, + }; + var input = DbRequestSummary( + "INSERT INTO Users (Name, Token, ApiKey) VALUES (@name, @token, @apikey)", + ("@name", "Alice"), ("@token", "secret-xyz"), ("@apikey", "k-987")); + var evt = NewEvent( + channel: AuditChannel.DbOutbound, request: input, target: "PrimaryDb.INSERT INTO Users"); + + var result = Redactor(opts).Apply(evt); + + var d = Details(result); + Assert.Contains("\"@name\":\"Alice\"", d.RequestSummary!); + Assert.Contains("\"@token\":\"\"", d.RequestSummary!); + Assert.Contains("\"@apikey\":\"\"", d.RequestSummary!); + Assert.DoesNotContain("secret-xyz", d.RequestSummary!); + Assert.DoesNotContain("k-987", d.RequestSummary!); + } + + [Fact] + public void NonDbOutboundChannel_NotAffected() + { + var opts = new AuditLogOptions + { + PerTargetOverrides = new Dictionary + { + ["PrimaryDb"] = new PerTargetRedactionOverride { RedactSqlParamsMatching = "^@token$" }, + }, + }; + var input = DbRequestSummary("SELECT @token", ("@token", "should-survive")); + // ApiOutbound channel whose summary *looks* like the DbOutbound shape. + var evt = NewEvent( + channel: AuditChannel.ApiOutbound, request: input, target: "PrimaryDb.SELECT"); + + var result = Redactor(opts).Apply(evt); + + Assert.Equal(input, Details(result).RequestSummary); + } + + // ---- Truncation + cap selection (ports TruncationTests) ---------------- + + [Fact] + public void SuccessRow_10KB_RequestSummary_TruncatedTo8KB_PayloadTruncatedTrue() + { + var evt = NewEvent( + channel: AuditChannel.ApiOutbound, status: AuditStatus.Delivered, + outcome: AuditOutcome.Success, request: new string('a', 10 * 1024)); + + var result = Redactor().Apply(evt); + + var d = Details(result); + Assert.Equal(8192, Encoding.UTF8.GetByteCount(d.RequestSummary!)); + Assert.True(d.PayloadTruncated); + } + + [Fact] + public void ErrorRow_Failed_10KB_RequestSummary_NotTruncated_UnderErrorCap() + { + var evt = NewEvent( + channel: AuditChannel.ApiOutbound, status: AuditStatus.Failed, + outcome: AuditOutcome.Failure, request: new string('b', 10 * 1024)); + + var result = Redactor().Apply(evt); + + var d = Details(result); + Assert.Equal(new string('b', 10 * 1024), d.RequestSummary); + Assert.False(d.PayloadTruncated); + } + + [Fact] + public void ErrorRow_Failed_70KB_RequestSummary_TruncatedTo64KB() + { + var evt = NewEvent( + channel: AuditChannel.ApiOutbound, status: AuditStatus.Failed, + outcome: AuditOutcome.Failure, request: new string('c', 70 * 1024)); + + var result = Redactor().Apply(evt); + + var d = Details(result); + Assert.Equal(65536, Encoding.UTF8.GetByteCount(d.RequestSummary!)); + Assert.True(d.PayloadTruncated); + } + + [Fact] + public void StatusAttempted_TreatedAsError_UsesErrorCap_EvenThoughOutcomeSuccess() + { + // Attempted projects to Outcome.Success, yet IsErrorStatus(Attempted)==true. + // Faithful port must read d.Status and pick the error cap — a 10 KB body + // must survive (under the 64 KiB error cap), NOT truncate to 8 KiB. + var evt = NewEvent( + channel: AuditChannel.ApiOutbound, status: AuditStatus.Attempted, + outcome: AuditOutcome.Success, request: new string('d', 10 * 1024)); + + var result = Redactor().Apply(evt); + + var d = Details(result); + Assert.Equal(new string('d', 10 * 1024), d.RequestSummary); + Assert.False(d.PayloadTruncated); + } + + [Fact] + public void StatusSkipped_TreatedAsError_UsesErrorCap_EvenThoughOutcomeSuccess() + { + var evt = NewEvent( + channel: AuditChannel.ApiOutbound, status: AuditStatus.Skipped, + outcome: AuditOutcome.Success, request: new string('f', 10 * 1024)); + + var result = Redactor().Apply(evt); + + Assert.False(Details(result).PayloadTruncated); + } + + [Fact] + public void StatusSubmitted_TreatedAsSuccess_UsesDefaultCap() + { + // Submitted is NOT an error status (IsErrorStatus==false) → default 8 KiB cap. + var evt = NewEvent( + channel: AuditChannel.ApiOutbound, status: AuditStatus.Submitted, + outcome: AuditOutcome.Success, request: new string('g', 10 * 1024)); + + var result = Redactor().Apply(evt); + + var d = Details(result); + Assert.Equal(8192, Encoding.UTF8.GetByteCount(d.RequestSummary!)); + Assert.True(d.PayloadTruncated); + } + + [Fact] + public void ApiInbound_LargeBody_UsesInboundCap_NotDefault() + { + var evt = NewEvent( + channel: AuditChannel.ApiInbound, status: AuditStatus.Delivered, + outcome: AuditOutcome.Success, request: new string('a', 100_000)); + + var result = Redactor().Apply(evt); + + var d = Details(result); + Assert.False(d.PayloadTruncated); + Assert.Equal(100_000, Encoding.UTF8.GetByteCount(d.RequestSummary!)); + } + + [Fact] + public void ApiInbound_Failed_BodyAboveInboundMaxBytes_TruncatedToInboundMaxBytes() + { + var opts = new AuditLogOptions { InboundMaxBytes = 16_384 }; + var evt = NewEvent( + channel: AuditChannel.ApiInbound, status: AuditStatus.Failed, + outcome: AuditOutcome.Failure, response: new string('z', 50_000)); + + var result = Redactor(opts).Apply(evt); + + var d = Details(result); + Assert.True(d.PayloadTruncated); + Assert.True(Encoding.UTF8.GetByteCount(d.ResponseSummary!) <= 16_384); + } + + [Fact] + public void Multibyte_UTF8_TruncatedAtCharacterBoundary_NotMidByte() + { + var sb = new StringBuilder(); + for (int i = 0; i < 2100; i++) sb.Append("😀"); + var input = sb.ToString(); + var evt = NewEvent(request: input); + + var result = Redactor().Apply(evt); + + var d = Details(result); + var bytes = Encoding.UTF8.GetByteCount(d.RequestSummary!); + Assert.True(bytes <= 8192); + Assert.Equal(0, bytes % 4); + Assert.DoesNotContain('�', d.RequestSummary!); + Assert.True(d.PayloadTruncated); + } + + [Fact] + public void ExistingDetailsPayloadTruncated_RemainsTrue() + { + var evt = NewEvent(request: "small", detailsPayloadTruncated: true); + + var result = Redactor().Apply(evt); + + var d = Details(result); + Assert.Equal("small", d.RequestSummary); + Assert.True(d.PayloadTruncated); + } + + // ---- Target length cap ------------------------------------------------- + + [Fact] + public void Target_OverCap_Truncated_ToByteBoundary() + { + // Target is a canonical top-level field; the redactor caps it at the + // default cap so an absurdly long target can't blow the column. + var longTarget = new string('t', 10 * 1024); + var evt = NewEvent(status: AuditStatus.Delivered, outcome: AuditOutcome.Success, target: longTarget); + + var result = Redactor().Apply(evt); + + Assert.NotNull(result.Target); + Assert.True(Encoding.UTF8.GetByteCount(result.Target!) <= 8192); + } + + // ---- Fast-path --------------------------------------------------------- + + [Fact] + public void FastPath_NullDetailsAndShortTarget_ReturnsSameInstance() + { + var evt = new AuditEvent + { + EventId = Guid.NewGuid(), + OccurredAtUtc = DateTimeOffset.UtcNow, + Actor = "tester", + Action = "ApiOutbound.ApiCall", + Outcome = AuditOutcome.Success, + Target = "short", + DetailsJson = null, + }; + + var result = Redactor().Apply(evt); + + Assert.Same(evt, result); + } + + [Fact] + public void FastPath_EmptyDetailsAndShortTarget_ReturnsSameInstance() + { + var evt = new AuditEvent + { + EventId = Guid.NewGuid(), + OccurredAtUtc = DateTimeOffset.UtcNow, + Actor = "tester", + Action = "ApiOutbound.ApiCall", + Outcome = AuditOutcome.Success, + Target = null, + DetailsJson = "", + }; + + var result = Redactor().Apply(evt); + + Assert.Same(evt, result); + } + + // ---- Never-throws safety net ------------------------------------------- + + [Fact] + public void MalformedDetailsJson_NeverThrows_ReturnsSafeCopy() + { + // Deserialize never throws (returns empty details), but a malformed + // DetailsJson with a long Target still flows the slow path. Assert the + // redactor returns a result without throwing. + var evt = new AuditEvent + { + EventId = Guid.NewGuid(), + OccurredAtUtc = DateTimeOffset.UtcNow, + Actor = "tester", + Action = "ApiOutbound.ApiCall", + Outcome = AuditOutcome.Success, + Target = new string('x', 10 * 1024), + DetailsJson = "{not valid json at all]", + }; + + var ex = Record.Exception(() => Redactor().Apply(evt)); + + Assert.Null(ex); + } + + /// Counts calls. + private sealed class CountingRedactionFailureCounter : IAuditRedactionFailureCounter + { + private int _count; + public int Count => _count; + public void Increment() => System.Threading.Interlocked.Increment(ref _count); + } + + /// IOptionsMonitor test double — returns the same snapshot on every read. + private sealed class StaticMonitor : IOptionsMonitor + { + private readonly AuditLogOptions _value; + public StaticMonitor(AuditLogOptions value) => _value = value; + public AuditLogOptions CurrentValue => _value; + public AuditLogOptions Get(string? name) => _value; + public IDisposable? OnChange(Action listener) => null; + } +}