using System.Text; using System.Text.RegularExpressions; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using ZB.MOM.WW.Audit; using ZB.MOM.WW.ScadaBridge.AuditLog.Configuration; using ZB.MOM.WW.ScadaBridge.AuditLog.Payload; using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; namespace ZB.MOM.WW.ScadaBridge.AuditLog.Redaction; /// /// Canonical implementation for ScadaBridge — /// operates on ZB.MOM.WW.Audit.AuditEvent and its /// payload bag. The ScadaBridge request/response/error/extra summaries travel /// inside DetailsJson as a record (serialized /// by ); this redactor deserializes them, applies /// the header → body-regex → SQL-parameter → byte-safe truncation pipeline, /// re-serializes, and returns a filtered COPY. /// /// /// /// Cap selection is faithful to the original pipeline, translated onto canonical /// fields: /// /// The ApiInbound branch keys on /// (= AuditChannel.ToString() per ) /// → . /// The "error row" branch reproduces the legacy /// IsErrorStatus(Status) rule — Status NOT IN (Delivered, /// Submitted, Forwarded) → . /// The fine-grained status is read from /// when present (it must be — alone cannot /// reproduce IsErrorStatus, since Attempted/Skipped /// project to yet take the error cap). /// When is absent/unparseable the /// canonical is the fallback: /// / /// → error cap. /// /// /// /// MUST NOT throw — wrapped in try/catch; over-redacts (drops ALL sensitive free-text /// fields to a safe marker) on any internal failure, mirroring /// . /// /// public sealed class ScadaBridgeAuditRedactor : IAuditRedactor, IDisposable { private const string OverRedactedMarker = AuditRedactionPrimitives.OverRedactedEventMarker; private readonly IOptionsMonitor _options; private readonly ILogger _logger; private readonly IAuditRedactionFailureCounter _failureCounter; private readonly AuditRegexCache _regexCache; private readonly IDisposable? _optionsReload; /// /// Primary constructor used by DI — pulls the optional redaction-failure /// counter from the container; a NoOp default is used when none is supplied. /// /// Live-reloadable audit log options. /// Logger for redaction diagnostics. /// Optional counter incremented when a redaction operation fails; defaults to a no-op. public ScadaBridgeAuditRedactor( IOptionsMonitor options, ILogger logger, IAuditRedactionFailureCounter? failureCounter = null) { _options = options ?? throw new ArgumentNullException(nameof(options)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _failureCounter = failureCounter ?? new NoOpAuditRedactionFailureCounter(); _regexCache = new AuditRegexCache(_logger); // Precompile at construction and on every reload, per the audit-log // roadmap ("patterns precompiled at startup; rejected if compile takes // >100ms"). Compiling lazily on first event instead put a wall-clock // budget on the hot path, where RegexOptions.Compiled IL emission can // exceed it under load and reject a valid pattern. WarmCurrentPatterns(); _optionsReload = _options.OnChange(WarmRedactorPatterns); } /// /// Warm from the current options snapshot. Reading CurrentValue happens /// INSIDE the try: an options provider that throws must not take the /// constructor down — owns that failure and answers it by /// over-redacting, which is the behaviour /// OuterCatch_OptionsThrows_NeverLeaks_AllSensitiveFieldsOverRedacted /// pins. /// private void WarmCurrentPatterns() { try { WarmRedactorPatterns(_options.CurrentValue); } catch (Exception ex) { _logger.LogWarning( ex, "Could not read audit options to precompile redactor patterns; deferring to first use"); } } /// /// Compile every configured body / SQL-parameter redactor pattern so the /// compile budget is spent once, off the audit hot path. /// /// /// Warming is best-effort and deliberately does NOT fail the boot: an /// unusable pattern degrades that node to over-redaction (safe, loud) rather /// than taking the node down. There is still a narrow window after a reload /// where an event can beat the warm-up and compile on the hot path — that /// path now fails closed, so the worst case is over-redaction, never a /// silently unredacted payload. /// private void WarmRedactorPatterns(AuditLogOptions? opts) { if (opts == null) { return; } try { var unavailable = _regexCache.Warm(opts.GlobalBodyRedactors); foreach (var over in opts.PerTargetOverrides.Values) { unavailable += _regexCache.Warm(over.AdditionalBodyRedactors); if (!string.IsNullOrEmpty(over.RedactSqlParamsMatching)) { unavailable += _regexCache.Warm(new[] { "(?i)" + over.RedactSqlParamsMatching }); } } if (unavailable > 0) { _logger.LogWarning( "{Count} configured audit redactor pattern(s) are unavailable; audit payloads " + "matching them will be over-redacted until the patterns are fixed.", unavailable); } } catch (Exception ex) { // Never let warm-up break construction or an options reload; the // lazy path remains as the (fail-closed) fallback. _logger.LogWarning(ex, "Audit redactor pattern warm-up failed; patterns will compile on first use"); } } /// Disposes the options-change registration, if any. public void Dispose() => _optionsReload?.Dispose(); /// /// Applies the full redaction pipeline to and returns a /// filtered copy; returns the same instance unchanged on the fast path. Never throws. /// /// The raw audit event to redact. /// A redacted copy of , or the original instance when no changes are needed. public AuditEvent Apply(AuditEvent rawEvent) { try { var opts = _options.CurrentValue; // --- Fast path ------------------------------------------------- // Mirror the legacy filter's non-JSON pre-check: when there is no // DetailsJson payload to scrub AND the Target is within the cap, // there is nothing to redact or truncate. Return the input // unchanged so the common case stays cheap (no Deserialize, no // re-Serialize, same instance back). var detailsEmpty = string.IsNullOrEmpty(rawEvent.DetailsJson); var targetWithinCap = rawEvent.Target is null || Encoding.UTF8.GetByteCount(rawEvent.Target) <= opts.DefaultCapBytes; if (detailsEmpty && targetWithinCap) { return rawEvent; } // --- Slow path ------------------------------------------------- var d = AuditDetailsCodec.Deserialize(rawEvent.DetailsJson); // Cap selection. Channel = canonical Category (the ApiInbound // branch); error-cap selection reproduces the legacy // IsErrorStatus(Status) — read from d.Status when present, else // fall back to the canonical Outcome. var cap = SelectCap(opts, rawEvent.Category, d.Status, rawEvent.Outcome); // --- Header-redaction stage (runs BEFORE truncation) ---------- var request = RedactHeaders(d.RequestSummary, opts.HeaderRedactList); var response = RedactHeaders(d.ResponseSummary, opts.HeaderRedactList); var errorDetail = d.ErrorDetail; var errorMessage = d.ErrorMessage; var extra = d.Extra; // --- Body-regex stage (also runs BEFORE truncation) ----------- // Per-target additions key on the canonical Target. var bodyRegexes = ResolveBodyRegexes(opts, rawEvent.Target, out var bodyRedactorUnavailable); // FAIL CLOSED. A configured redactor that will not compile must never // degrade to "emit the body raw" — that publishes precisely the values // the operator asked to suppress, on a row that looks entirely normal. if (bodyRedactorUnavailable) { return OverRedactUnavailable(rawEvent, "body"); } if (bodyRegexes.Count > 0) { request = RedactBody(request, bodyRegexes); response = RedactBody(response, bodyRegexes); errorDetail = RedactBody(errorDetail, bodyRegexes); errorMessage = RedactBody(errorMessage, bodyRegexes); extra = RedactBody(extra, bodyRegexes); } // --- SQL parameter redaction stage (DbOutbound only) ---------- // Channel-guarded on the canonical Category; connection key is the // Target prefix before the first '.'. if (string.Equals(rawEvent.Category, nameof(AuditChannel.DbOutbound), StringComparison.Ordinal)) { if (TryGetSqlParamRedactor( opts, rawEvent.Target, out var sqlParamRegex, out var sqlRedactorUnavailable)) { request = RedactSqlParameters(request, sqlParamRegex!); // Parameter policy matches names, whereas SQL exceptions can echo VALUES. // Free text cannot be safely filtered with a name regex. Keep status/category. if (errorMessage is not null) errorMessage = "Database operation failed (sensitive SQL error suppressed)"; if (errorDetail is not null) errorDetail = ""; } else if (sqlRedactorUnavailable) { return OverRedactUnavailable(rawEvent, "SQL-parameter"); } } // --- Truncation stage ----------------------------------------- var truncated = false; request = TruncateField(request, cap, ref truncated); response = TruncateField(response, cap, ref truncated); errorDetail = TruncateField(errorDetail, cap, ref truncated); errorMessage = TruncateField(errorMessage, cap, ref truncated); extra = TruncateField(extra, cap, ref truncated); var rewritten = d with { RequestSummary = request, ResponseSummary = response, ErrorDetail = errorDetail, ErrorMessage = errorMessage, Extra = extra, PayloadTruncated = d.PayloadTruncated || truncated, }; // Target length cap (canonical top-level field). Cap at the default // byte ceiling so an absurd Target cannot blow the storage column. var cappedTarget = TruncateTarget(rawEvent.Target, opts.DefaultCapBytes); return rawEvent with { DetailsJson = AuditDetailsCodec.Serialize(rewritten), Target = cappedTarget, }; } catch (Exception ex) { // Audit is best-effort: over-redact rather than fail the caller. // Drop the summaries entirely (mirroring SafeDefault's catch path) // and flag PayloadTruncated so downstream readers know the row was // scrubbed defensively. _logger.LogWarning( ex, "Canonical audit redactor failed; over-redacting DetailsJson and flagging PayloadTruncated"); IncrementFailureCounter(); return OverRedact(rawEvent); } } /// /// Pick the truncation cap. = canonical Category /// (= channel name): ApiInbound → . /// Otherwise the legacy IsErrorStatus rule decides between the error /// and default caps, preferring the fine-grained /// (from DetailsJson) and falling back to the canonical /// when status is absent/unparseable. /// private static int SelectCap( AuditLogOptions opts, string? category, string? detailsStatus, AuditOutcome outcome) { if (string.Equals(category, nameof(AuditChannel.ApiInbound), StringComparison.Ordinal)) { return opts.InboundMaxBytes; } return IsErrorRow(detailsStatus, outcome) ? opts.ErrorCapBytes : opts.DefaultCapBytes; } /// /// Reproduce the legacy IsErrorStatus(Status) error-cap predicate on /// the canonical record: Status NOT IN (Delivered, Submitted, /// Forwarded) → error row. When the fine-grained status is present in /// DetailsJson it is authoritative; otherwise the canonical /// is the fallback /// (/ /// → error row). /// private static bool IsErrorRow(string? detailsStatus, AuditOutcome outcome) { if (!string.IsNullOrEmpty(detailsStatus) && Enum.TryParse(detailsStatus, ignoreCase: false, out var status)) { return status switch { AuditStatus.Delivered or AuditStatus.Submitted or AuditStatus.Forwarded => false, _ => true, }; } // No usable status — fall back to the canonical outcome. return outcome != AuditOutcome.Success; } /// /// Fail-closed exit: a redactor the operator configured is unavailable, so the /// payload is suppressed wholesale rather than emitted unredacted. Counted on /// the same as other redaction /// failures so a disabled redactor is observable, not just a log line. /// private AuditEvent OverRedactUnavailable(AuditEvent rawEvent, string stage) { _logger.LogWarning( "A configured {Stage} redactor is unavailable (invalid or over the compile budget); " + "over-redacting DetailsJson rather than emitting it unredacted. " + "Fix or remove the pattern — audit payloads stay suppressed until then.", stage); IncrementFailureCounter(); return OverRedact(rawEvent); } private string? RedactHeaders(string? json, IList redactList) => AuditRedactionPrimitives.RedactHeaders(json, redactList, _logger, IncrementFailureCounter); private string? RedactBody(string? value, IReadOnlyList regexes) => AuditRedactionPrimitives.RedactBody(value, regexes, _logger, IncrementFailureCounter); private string? RedactSqlParameters(string? json, Regex paramNameRegex) => AuditRedactionPrimitives.RedactSqlParameters(json, paramNameRegex, _logger, IncrementFailureCounter); private static string? TruncateField(string? value, int cap, ref bool truncated) => AuditRedactionPrimitives.TruncateField(value, cap, ref truncated); private static string? TruncateTarget(string? target, int cap) => target is null ? null : AuditRedactionPrimitives.TruncateUtf8(target, cap); /// /// Combine the global and per-target body-redactor lists, returning the /// compiled-regex set to apply. /// /// /// is set when a pattern the operator /// CONFIGURED could not be compiled (invalid, or over the cache's compile /// budget). The caller must then fail closed — dropping the pattern and /// emitting the body anyway would silently publish exactly the values the /// operator asked to have redacted. /// private IReadOnlyList ResolveBodyRegexes( AuditLogOptions opts, string? target, out bool anyUnavailable) { anyUnavailable = false; var hasGlobal = opts.GlobalBodyRedactors is { Count: > 0 }; var perTargetAdditions = (target != null && opts.PerTargetOverrides.TryGetValue(target, out var over) && over.AdditionalBodyRedactors is { Count: > 0 }) ? over.AdditionalBodyRedactors : null; if (!hasGlobal && perTargetAdditions == null) { return Array.Empty(); } var result = new List(); if (hasGlobal) { foreach (var pattern in opts.GlobalBodyRedactors) { if (_regexCache.TryGet(pattern, out var rx)) { result.Add(rx!); } else { anyUnavailable = true; } } } if (perTargetAdditions != null) { foreach (var pattern in perTargetAdditions) { if (_regexCache.TryGet(pattern, out var rx)) { result.Add(rx!); } else { anyUnavailable = true; } } } return result; } /// /// Resolve the per-connection SQL parameter redaction regex for the given /// target. Connection key = everything before the first . in /// . Patterns are forced case-insensitive. /// /// /// separates "no SQL redactor is configured for /// this connection" (fine — return false, capture parameters as normal) from /// "one IS configured but will not compile" (fail closed). Collapsing the two /// into a bare false is what let a configured redactor silently emit /// parameter values verbatim. /// private bool TryGetSqlParamRedactor( AuditLogOptions opts, string? target, out Regex? regex, out bool unavailable) { regex = null; unavailable = false; if (string.IsNullOrEmpty(target)) { return false; } var dot = target.IndexOf('.'); var connectionKey = dot < 0 ? target : target[..dot]; if (!opts.PerTargetOverrides.TryGetValue(connectionKey, out var over) || string.IsNullOrEmpty(over.RedactSqlParamsMatching)) { return false; } var cacheKey = "(?i)" + over.RedactSqlParamsMatching; if (_regexCache.TryGet(cacheKey, out regex)) { return true; } unavailable = true; return false; } /// /// Over-redaction copy returned from the never-throws catch: suppress ALL /// potentially-sensitive string fields inside DetailsJson to a safe /// marker and flag . "All sensitive /// fields" = RequestSummary, ResponseSummary, ErrorDetail, /// ErrorMessage, and Extra — all body-regex redaction targets /// that can carry sensitive values. Best-effort re-serialise; if even that /// fails, return the input with no sensitive fields via a minimal details bag. /// private static AuditEvent OverRedact(AuditEvent rawEvent) { try { var d = AuditDetailsCodec.Deserialize(rawEvent.DetailsJson) with { RequestSummary = OverRedactedMarker, ResponseSummary = OverRedactedMarker, ErrorDetail = OverRedactedMarker, ErrorMessage = OverRedactedMarker, Extra = OverRedactedMarker, PayloadTruncated = true, }; return rawEvent with { DetailsJson = AuditDetailsCodec.Serialize(d) }; } catch { var safe = new AuditDetails { RequestSummary = OverRedactedMarker, ResponseSummary = OverRedactedMarker, ErrorDetail = OverRedactedMarker, ErrorMessage = OverRedactedMarker, Extra = OverRedactedMarker, PayloadTruncated = true, }; return rawEvent with { DetailsJson = AuditDetailsCodec.Serialize(safe) }; } } /// /// Bumps the injected redaction-failure counter, swallowing any fault per /// alog.md §7. Passed as the onFailure callback to the shared /// primitives and called from the top-level catch. /// private void IncrementFailureCounter() { try { _failureCounter.Increment(); } catch { /* swallow per §7 */ } } }