dohertj2/ScadaBridge
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(audit)!: ScadaBridge C3 — swap to canonical ZB.MOM.WW.Audit.AuditEvent across seams/emitters/DTO/redactor wiring; transitional 24-col storage shim (Task 2.5)

Browse Source
This commit is contained in:
Joseph Doherty
2026-06-02 12:37:50 -04:00
parent 5aaf9e2923
commit db707bb0de
127 changed files with 2240 additions and 3886 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.AuditLog/Payload/DefaultAuditPayloadFilter.cs
-304
View File
@@ -1,304 +0,0 @@
using System.Text.RegularExpressions;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.AuditLog.Configuration;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Payload;
/// <summary>
/// Default <see cref="IAuditPayloadFilter"/>. Bundle A established the
/// truncation backbone; Bundle B chains HTTP header redaction (M5-T3) BEFORE
/// truncation so redactors operate on the full payload and the cap then trims
/// the redacted result.
/// </summary>
/// <remarks>
/// <para>
/// Uses <see cref="IOptionsMonitor{TOptions}"/> (not <see cref="IOptions{TOptions}"/>)
/// so the M5-T8 hot-reload path sees fresh values without re-resolving the
/// singleton. <see cref="Apply"/> reads <see cref="IOptionsMonitor{T}.CurrentValue"/>
/// on every call, and the regex cache is keyed by pattern string — patterns
/// added via a live config change compile on first use of the next event;
/// patterns removed simply stop being looked up. No <c>OnChange</c> subscription
/// or explicit cache invalidation is required (the
/// <c>AuditLogOptionsBindingTests</c> fixture in <c>ZB.MOM.WW.ScadaBridge.AuditLog.Tests</c>
/// pins this behaviour).
/// </para>
/// <para>
/// "Error row" = <see cref="AuditEvent.Status"/> NOT IN (<c>Delivered</c>,
/// <c>Submitted</c>, <c>Forwarded</c>) — every other status, including the
/// non-terminal <c>Attempted</c>, the parked/discarded terminals, and the
/// short-circuit <c>Skipped</c>, receives the larger error cap so a verbose
/// error body survives.
/// </para>
/// <para>
/// Apply MUST NOT throw — on internal failure the filter over-redacts by
/// returning the input with <see cref="AuditEvent.PayloadTruncated"/> set and
/// increments the <c>AuditRedactionFailure</c> health metric via the injected
/// <see cref="IAuditRedactionFailureCounter"/>. Each redactor stage runs in
/// its own try/catch — a failure in (say) the header redactor still lets the
/// SQL parameter redactor and the truncator run on the remaining fields.
/// </para>
/// <para>
/// Stage order (each runs on every applicable field):
/// header redaction → body regex redaction → truncation. The SQL-parameter
/// stage piggybacks on the body-redactor path; both run BEFORE truncation so
/// the cap trims the redacted result, never bytes the redactor intended to
/// hide.
/// </para>
/// </remarks>
public sealed class DefaultAuditPayloadFilter : IAuditPayloadFilter
{
// Redaction markers + the relaxed-escaping JSON options live in
// AuditRedactionPrimitives, and the compiled-regex cache (50 ms match
// timeout, 100 ms compile budget, invalid-pattern sentinel) lives in
// AuditRegexCache — both shared C2 helpers so the legacy filter and the
// canonical ScadaBridgeAuditRedactor emit byte-identical output.
private readonly IOptionsMonitor<AuditLogOptions> _options;
private readonly ILogger<DefaultAuditPayloadFilter> _logger;
private readonly IAuditRedactionFailureCounter _failureCounter;
private readonly AuditRegexCache _regexCache;
/// <summary>
/// Primary constructor used by DI — pulls the optional redaction-failure
/// counter from the container; a NoOp default is registered in
/// <see cref="ServiceCollectionExtensions.AddAuditLog"/>.
/// </summary>
/// <param name="options">Live-reloadable audit log options.</param>
/// <param name="logger">Logger for redaction diagnostics.</param>
/// <param name="failureCounter">Optional counter incremented when a redaction operation fails; defaults to a no-op.</param>
public DefaultAuditPayloadFilter(
IOptionsMonitor<AuditLogOptions> options,
ILogger<DefaultAuditPayloadFilter> logger,
IAuditRedactionFailureCounter? failureCounter = null)
{
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_failureCounter = failureCounter ?? new NoOpAuditRedactionFailureCounter();
_regexCache = new AuditRegexCache(_logger);
}
/// <inheritdoc />
public AuditEvent Apply(AuditEvent rawEvent)
{
try
{
var opts = _options.CurrentValue;
// Inbound API gets a dedicated, larger ceiling — request/response bodies are
// captured verbatim up to InboundMaxBytes (default 1 MiB) so support can
// replay exactly what the caller sent and what we returned. Other channels
// keep the global 8 KiB / 64 KiB policy.
// See docs/plans/2026-05-23-inbound-api-full-response-audit-design.md.
var cap = rawEvent.Channel == AuditChannel.ApiInbound
? opts.InboundMaxBytes
: (IsErrorStatus(rawEvent.Status) ? opts.ErrorCapBytes : opts.DefaultCapBytes);
// --- Header-redaction stage (runs BEFORE truncation) ----------
var request = RedactHeaders(rawEvent.RequestSummary, opts.HeaderRedactList);
var response = RedactHeaders(rawEvent.ResponseSummary, opts.HeaderRedactList);
var errorDetail = rawEvent.ErrorDetail;
var extra = rawEvent.Extra;
// --- Body-regex stage (also runs BEFORE truncation) -----------
// Resolves the active regex set per event so per-target overrides
// bound to AuditEvent.Target are picked up; effectively a no-op
// when neither GlobalBodyRedactors nor the per-target additions
// are configured.
var bodyRegexes = ResolveBodyRegexes(opts, rawEvent.Target);
if (bodyRegexes.Count > 0)
{
request = RedactBody(request, bodyRegexes);
response = RedactBody(response, bodyRegexes);
errorDetail = RedactBody(errorDetail, bodyRegexes);
extra = RedactBody(extra, bodyRegexes);
}
// --- SQL parameter redaction stage (DbOutbound only) ----------
// Parses the M4 AuditingDbCommand RequestSummary shape
// {"sql":"...","parameters":{...}} and redacts parameter VALUES
// whose NAME matches the per-connection regex. Opt-in: no
// PerTargetOverrides[connectionName].RedactSqlParamsMatching =>
// no-op. Channel-guarded so the same regex can never accidentally
// touch an ApiOutbound row.
if (rawEvent.Channel == AuditChannel.DbOutbound
&& TryGetSqlParamRedactor(opts, rawEvent.Target, out var sqlParamRegex))
{
request = RedactSqlParameters(request, sqlParamRegex!);
}
// --- Truncation stage -----------------------------------------
var truncated = false;
request = TruncateField(request, cap, ref truncated);
response = TruncateField(response, cap, ref truncated);
errorDetail = TruncateField(errorDetail, cap, ref truncated);
extra = TruncateField(extra, cap, ref truncated);
return rawEvent with
{
RequestSummary = request,
ResponseSummary = response,
ErrorDetail = errorDetail,
Extra = extra,
PayloadTruncated = rawEvent.PayloadTruncated || truncated,
};
}
catch (Exception ex)
{
// Audit is best-effort: over-redact rather than fail the caller.
// The per-stage try/catches above already handle redactor faults
// and increment the counter; this catch covers any unexpected
// surprise in the surrounding orchestration code.
_logger.LogWarning(
ex,
"Payload filter failed; returning raw event with PayloadTruncated=true");
try { _failureCounter.Increment(); } catch { /* swallow per §7 */ }
return rawEvent with { PayloadTruncated = true };
}
}
/// <summary>
/// Parse <paramref name="json"/> as the documented
/// <c>{"headers": {...}, "body": ...}</c> shape and replace values whose
/// header NAME (case-insensitive) is in <paramref name="redactList"/> with
/// the redaction marker. Re-serialises and returns the result. Delegates to
/// <see cref="AuditRedactionPrimitives.RedactHeaders"/>.
/// </summary>
/// <remarks>
/// No-op pass-through for inputs that aren't JSON-shaped — emitters that
/// have not yet adopted the convention (the M2 site emitters today, which
/// leave RequestSummary null on outbound API calls) get a transparent
/// pass. If the redactor itself throws, we over-redact the whole field
/// with the redactor-error marker and bump the failure counter.
/// </remarks>
private string? RedactHeaders(string? json, IList<string> redactList)
=> AuditRedactionPrimitives.RedactHeaders(json, redactList, _logger, IncrementFailureCounter);
/// <summary>
/// Combine the global and per-target body-redactor lists for a single
/// event, returning the compiled-regex set to apply. Patterns that failed
/// compilation are silently skipped — the compile-time failure was logged
/// once on first encounter; we never let one bad pattern starve the rest.
/// </summary>
private IReadOnlyList<Regex> ResolveBodyRegexes(AuditLogOptions opts, string? target)
{
var hasGlobal = opts.GlobalBodyRedactors is { Count: > 0 };
var perTargetAdditions = (target != null
&& opts.PerTargetOverrides.TryGetValue(target, out var over)
&& over.AdditionalBodyRedactors is { Count: > 0 })
? over.AdditionalBodyRedactors
: null;
if (!hasGlobal && perTargetAdditions == null)
{
return Array.Empty<Regex>();
}
var result = new List<Regex>();
if (hasGlobal)
{
foreach (var pattern in opts.GlobalBodyRedactors)
{
if (_regexCache.TryGet(pattern, out var rx))
{
result.Add(rx!);
}
}
}
if (perTargetAdditions != null)
{
foreach (var pattern in perTargetAdditions)
{
if (_regexCache.TryGet(pattern, out var rx))
{
result.Add(rx!);
}
}
}
return result;
}
/// <summary>
/// Apply each compiled body-redactor regex to <paramref name="value"/> in
/// turn, replacing every match with the redaction marker. If any single
/// regex match throws (most commonly
/// <see cref="RegexMatchTimeoutException"/>) the field is over-redacted with
/// the redactor-error marker and the failure counter is incremented — the
/// user-facing action is never aborted. Delegates to
/// <see cref="AuditRedactionPrimitives.RedactBody"/>.
/// </summary>
private string? RedactBody(string? value, IReadOnlyList<Regex> regexes)
=> AuditRedactionPrimitives.RedactBody(value, regexes, _logger, IncrementFailureCounter);
/// <summary>
/// Resolve the per-connection SQL parameter redaction regex for the given
/// DbOutbound event target. Target shape (M4 AuditingDbCommand): the
/// connection name optionally followed by <c>.&lt;sql-snippet&gt;</c> for
/// disambiguation; the per-target dictionary is keyed by the connection
/// name alone, so we strip the snippet suffix before lookup. Patterns are
/// compiled with case-insensitive matching to match the documented
/// behaviour.
/// </summary>
private bool TryGetSqlParamRedactor(AuditLogOptions opts, string? target, out Regex? regex)
{
regex = null;
if (string.IsNullOrEmpty(target))
{
return false;
}
var dot = target.IndexOf('.');
var connectionKey = dot < 0 ? target : target[..dot];
if (!opts.PerTargetOverrides.TryGetValue(connectionKey, out var over)
|| string.IsNullOrEmpty(over.RedactSqlParamsMatching))
{
return false;
}
// Force case-insensitivity per the spec — even if the operator wrote
// the pattern without an IgnoreCase flag. The compile cache key folds
// the option to keep the entries unambiguous.
var cacheKey = "(?i)" + over.RedactSqlParamsMatching;
if (!_regexCache.TryGet(cacheKey, out regex))
{
return false;
}
return true;
}
/// <summary>
/// Walk the M4 <c>{"sql":"...","parameters":{...}}</c> RequestSummary
/// shape; for each parameter whose NAME matches
/// <paramref name="paramNameRegex"/>, replace its value with the redaction
/// marker. Re-serialise. Delegates to
/// <see cref="AuditRedactionPrimitives.RedactSqlParameters"/>.
/// </summary>
/// <remarks>
/// No-op pass-through when the input isn't parseable JSON, isn't a JSON
/// object, or doesn't carry a top-level <c>"parameters"</c> object. On
/// any unexpected fault the field is over-redacted and the failure counter
/// is bumped.
/// </remarks>
private string? RedactSqlParameters(string? json, Regex paramNameRegex)
=> AuditRedactionPrimitives.RedactSqlParameters(json, paramNameRegex, _logger, IncrementFailureCounter);
private static string? TruncateField(string? value, int cap, ref bool truncated)
=> AuditRedactionPrimitives.TruncateField(value, cap, ref truncated);
/// <summary>
/// Bumps the injected redaction-failure counter, swallowing any fault per
/// alog.md §7 (a counter failure must never abort the audited action).
/// Passed as the <c>onFailure</c> callback to the shared primitives.
/// </summary>
private void IncrementFailureCounter()
{
try { _failureCounter.Increment(); } catch { /* swallow per §7 */ }
}
private static bool IsErrorStatus(AuditStatus status) => status switch
{
AuditStatus.Delivered or AuditStatus.Submitted or AuditStatus.Forwarded => false,
_ => true,
};
}
src/ZB.MOM.WW.ScadaBridge.AuditLog/Payload/IAuditPayloadFilter.cs
-31
View File
@@ -1,31 +0,0 @@
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Payload;
/// <summary>
/// Filters an <see cref="AuditEvent"/> between construction and persistence —
/// truncates oversized payload fields, applies header/body/SQL-parameter
/// redaction, sets <see cref="AuditEvent.PayloadTruncated"/>.
/// </summary>
/// <remarks>
/// <para>
/// Pure function: returns a filtered COPY of the input via <c>with</c>
/// expressions; never throws (over-redacts on internal failure and increments
/// the <c>AuditRedactionFailure</c> health metric).
/// </para>
/// <para>
/// Wired in M5 between event construction and the writer chain
/// (<c>FallbackAuditWriter.WriteAsync</c>, <c>CentralAuditWriter.WriteAsync</c>,
/// and the <c>AuditLogIngestActor</c> handlers).
/// </para>
/// </remarks>
public interface IAuditPayloadFilter
{
/// <summary>
/// Apply the configured truncation + redaction policy to <paramref name="rawEvent"/>
/// and return a filtered copy. MUST NOT throw — on internal failure, over-redact
/// and surface the failure via the audit-redaction-failure health metric.
/// </summary>
/// <param name="rawEvent">The unfiltered audit event to process.</param>
AuditEvent Apply(AuditEvent rawEvent);
}
src/ZB.MOM.WW.ScadaBridge.AuditLog/Payload/SafeDefaultAuditPayloadFilter.cs
-79
View File
@@ -1,79 +0,0 @@
using System.Text.RegularExpressions;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Payload;
/// <summary>
/// AuditLog-008: minimal always-safe fallback filter used by the writer chain
/// when no <see cref="IAuditPayloadFilter"/> is injected (test composition
/// roots, future composition roots that bypass <c>AddAuditLog</c>). Performs
/// HTTP header redaction for the always-sensitive defaults
/// (Authorization, X-Api-Key, Cookie, Set-Cookie) so a fixture that wires a
/// real <see cref="AuditEvent.RequestSummary"/> never persists those headers
/// in cleartext. Does NOT perform body-regex redaction, SQL-parameter
/// redaction, or truncation — those stages need
/// <see cref="DefaultAuditPayloadFilter"/> with live options. The contract is:
/// over-redact safely, never throw, never miss a header that's on the
/// default sensitive list.
/// </summary>
public sealed class SafeDefaultAuditPayloadFilter : IAuditPayloadFilter
{
/// <summary>Singleton instance — the filter is stateless and side-effect-free.</summary>
public static SafeDefaultAuditPayloadFilter Instance { get; } = new SafeDefaultAuditPayloadFilter();
private static readonly string[] DefaultHeaderRedactList =
{
"Authorization",
"X-Api-Key",
"Cookie",
"Set-Cookie",
};
private static readonly Regex HeaderRegex = new(
@"(?<name>[A-Za-z][A-Za-z0-9\-_]*)\s*:\s*(?<value>[^\r\n]*)",
RegexOptions.Compiled | RegexOptions.IgnoreCase);
private SafeDefaultAuditPayloadFilter() { }
/// <inheritdoc />
public AuditEvent Apply(AuditEvent rawEvent)
{
ArgumentNullException.ThrowIfNull(rawEvent);
try
{
return rawEvent with
{
RequestSummary = RedactHeaders(rawEvent.RequestSummary),
ResponseSummary = RedactHeaders(rawEvent.ResponseSummary),
};
}
catch
{
// Over-redact: drop both summaries entirely so a malformed parse
// path never leaks the original. The contract is "never throw."
return rawEvent with
{
RequestSummary = "[redacted by SafeDefaultAuditPayloadFilter]",
ResponseSummary = "[redacted by SafeDefaultAuditPayloadFilter]",
};
}
}
private static string? RedactHeaders(string? summary)
{
if (string.IsNullOrEmpty(summary)) return summary;
return HeaderRegex.Replace(summary, m =>
{
var name = m.Groups["name"].Value;
foreach (var sensitive in DefaultHeaderRedactList)
{
if (string.Equals(name, sensitive, StringComparison.OrdinalIgnoreCase))
{
return $"{name}: [REDACTED]";
}
}
return m.Value;
});
}
}