dohertj2/scadalink-design
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(auditlog): HTTP header redaction stage (#23 M5)

Browse Source
This commit is contained in:
Joseph Doherty
2026-05-20 17:07:01 -04:00
parent bba2ef1b4d
commit ad7b330f43
5 changed files with 402 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

154
src/ScadaLink.AuditLog/Payload/DefaultAuditPayloadFilter.cs
View File

@@ -1,4 +1,7 @@
using System.Text;
using System.Text.Encodings.Web;
using System.Text.Json;
using System.Text.Json.Nodes;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using ScadaLink.AuditLog.Configuration;
@@ -8,11 +11,10 @@ using ScadaLink.Commons.Types.Enums;
namespace ScadaLink.AuditLog.Payload;
/// <summary>
/// Default <see cref="IAuditPayloadFilter"/>. M5 Bundle A scope: payload
/// truncation only (RequestSummary / ResponseSummary / ErrorDetail / Extra),
/// capped at <see cref="AuditLogOptions.DefaultCapBytes"/> on success rows and
/// <see cref="AuditLogOptions.ErrorCapBytes"/> on error rows. Bundle B layers
/// header / body / SQL-parameter redaction on top.
/// Default <see cref="IAuditPayloadFilter"/>. Bundle A established the
/// truncation backbone; Bundle B chains HTTP header redaction (M5-T3) BEFORE
/// truncation so redactors operate on the full payload and the cap then trims
/// the redacted result.
/// </summary>
/// <remarks>
/// <para>
@@ -30,20 +32,54 @@ namespace ScadaLink.AuditLog.Payload;
/// <para>
/// Apply MUST NOT throw — on internal failure the filter over-redacts by
/// returning the input with <see cref="AuditEvent.PayloadTruncated"/> set and
/// (Bundle C) increments the <c>AuditRedactionFailure</c> health metric.
/// increments the <c>AuditRedactionFailure</c> health metric via the injected
/// <see cref="IAuditRedactionFailureCounter"/>. Each redactor stage runs in
/// its own try/catch — a failure in (say) the header redactor still lets the
/// SQL parameter redactor and the truncator run on the remaining fields.
/// </para>
/// <para>
/// Stage order (each runs on every applicable field):
/// header redaction → truncation. Bundle B will append body-regex and
/// SQL-parameter stages after header redaction and before truncation.
/// </para>
/// </remarks>
public sealed class DefaultAuditPayloadFilter : IAuditPayloadFilter
{
private const string RedactedMarker = "<redacted>";
private const string RedactorErrorMarker = "<redacted: redactor error>";
/// <summary>
/// JSON serializer options used to re-emit redacted summaries. The
/// UnsafeRelaxedJsonEscaping encoder is required so the redaction marker
/// (which contains <c>&lt;</c> / <c>&gt;</c>) survives unescaped — the
/// header-redaction tests grep for the literal marker, and the downstream
/// UI / log readers would rather see <c>&lt;redacted&gt;</c> than
/// <c><redacted></c>. The summaries are persisted to the audit
/// table and rendered in trusted-internal contexts only, so the relaxed
/// HTML-escaping rules do not introduce an XSS surface.
/// </summary>
private static readonly JsonSerializerOptions RedactedSummaryJsonOptions = new()
{
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
};
private readonly IOptionsMonitor<AuditLogOptions> _options;
private readonly ILogger<DefaultAuditPayloadFilter> _logger;
private readonly IAuditRedactionFailureCounter _failureCounter;
/// <summary>
/// Primary constructor used by DI — pulls the optional redaction-failure
/// counter from the container; a NoOp default is registered in
/// <see cref="ServiceCollectionExtensions.AddAuditLog"/>.
/// </summary>
public DefaultAuditPayloadFilter(
IOptionsMonitor<AuditLogOptions> options,
ILogger<DefaultAuditPayloadFilter> logger)
ILogger<DefaultAuditPayloadFilter> logger,
IAuditRedactionFailureCounter? failureCounter = null)
{
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_failureCounter = failureCounter ?? new NoOpAuditRedactionFailureCounter();
}
public AuditEvent Apply(AuditEvent rawEvent)
@@ -52,11 +88,20 @@ public sealed class DefaultAuditPayloadFilter : IAuditPayloadFilter
{
var opts = _options.CurrentValue;
var cap = IsErrorStatus(rawEvent.Status) ? opts.ErrorCapBytes : opts.DefaultCapBytes;
// --- Header-redaction stage (runs BEFORE truncation) ----------
var request = RedactHeaders(rawEvent.RequestSummary, opts.HeaderRedactList);
var response = RedactHeaders(rawEvent.ResponseSummary, opts.HeaderRedactList);
var errorDetail = rawEvent.ErrorDetail;
var extra = rawEvent.Extra;
// --- Truncation stage -----------------------------------------
var truncated = false;
var request = TruncateField(rawEvent.RequestSummary, cap, ref truncated);
var response = TruncateField(rawEvent.ResponseSummary, cap, ref truncated);
var errorDetail = TruncateField(rawEvent.ErrorDetail, cap, ref truncated);
var extra = TruncateField(rawEvent.Extra, cap, ref truncated);
request = TruncateField(request, cap, ref truncated);
response = TruncateField(response, cap, ref truncated);
errorDetail = TruncateField(errorDetail, cap, ref truncated);
extra = TruncateField(extra, cap, ref truncated);
return rawEvent with
{
RequestSummary = request,
@@ -69,14 +114,99 @@ public sealed class DefaultAuditPayloadFilter : IAuditPayloadFilter
catch (Exception ex)
{
// Audit is best-effort: over-redact rather than fail the caller.
// Bundle C wires the AuditRedactionFailure health metric here.
// The per-stage try/catches above already handle redactor faults
// and increment the counter; this catch covers any unexpected
// surprise in the surrounding orchestration code.
_logger.LogWarning(
ex,
"Payload filter failed; returning raw event with PayloadTruncated=true");
try { _failureCounter.Increment(); } catch { /* swallow per §7 */ }
return rawEvent with { PayloadTruncated = true };
}
}
/// <summary>
/// Parse <paramref name="json"/> as the documented
/// <c>{"headers": {...}, "body": ...}</c> shape and replace values whose
/// header NAME (case-insensitive) is in <paramref name="redactList"/> with
/// <see cref="RedactedMarker"/>. Re-serialises and returns the result.
/// </summary>
/// <remarks>
/// No-op pass-through for inputs that aren't JSON-shaped — emitters that
/// have not yet adopted the convention (the M2 site emitters today, which
/// leave RequestSummary null on outbound API calls) get a transparent
/// pass. If the redactor itself throws, we over-redact the whole field
/// with <see cref="RedactorErrorMarker"/> and bump the failure counter.
/// </remarks>
private string? RedactHeaders(string? json, IList<string> redactList)
{
if (json is null)
{
return null;
}
// Cheap structural pre-check: only attempt JSON parsing when the input
// actually looks like a JSON object. Saves the JsonDocument allocation
// on the (very common) non-JSON ErrorDetail / Extra fields.
var trimmed = json.AsSpan().TrimStart();
if (trimmed.Length == 0 || trimmed[0] != '{')
{
return json;
}
try
{
JsonNode? root;
try
{
root = JsonNode.Parse(json);
}
catch (JsonException)
{
// Not parseable JSON — leave the field alone (no error, no
// redaction). Emitters not yet using the documented shape get
// a transparent pass; Bundle C will update them.
return json;
}
if (root is not JsonObject obj || obj["headers"] is not JsonObject headers)
{
// No "headers" object at the top level — nothing to redact.
return json;
}
// Build a case-insensitive lookup of the redact list so we can do
// one O(1) check per header name without an inner Any() loop.
var redactSet = new HashSet<string>(redactList, StringComparer.OrdinalIgnoreCase);
// Take a snapshot of names first — we cannot mutate while
// enumerating the JsonObject.
var names = new List<string>(headers.Count);
foreach (var kvp in headers)
{
names.Add(kvp.Key);
}
foreach (var name in names)
{
if (redactSet.Contains(name))
{
headers[name] = JsonValue.Create(RedactedMarker);
}
}
return obj.ToJsonString(RedactedSummaryJsonOptions);
}
catch (Exception ex)
{
_logger.LogWarning(
ex,
"Header redactor faulted; over-redacting field with '{Marker}'",
RedactorErrorMarker);
try { _failureCounter.Increment(); } catch { /* swallow per §7 */ }
return RedactorErrorMarker;
}
}
private static string? TruncateField(string? value, int cap, ref bool truncated)
{
if (value is null)

20
src/ScadaLink.AuditLog/Payload/IAuditRedactionFailureCounter.cs Normal file
View File

@@ -0,0 +1,20 @@
namespace ScadaLink.AuditLog.Payload;
/// <summary>
/// Counter sink invoked by <see cref="DefaultAuditPayloadFilter"/> every time
/// a redactor (header / body regex / SQL parameter) throws and the filter has
/// to over-redact the offending field with the
/// <c>&lt;redacted: redactor error&gt;</c> marker. Bundle C bridges this into
/// the Site Health Monitoring report payload as <c>AuditRedactionFailure</c>.
/// </summary>
/// <remarks>
/// Redaction failures must NEVER abort the user-facing action (alog.md §7) —
/// the filter over-redacts the field and surfaces the failure via this counter
/// instead. A NoOp default is the correct safe fallback while the health
/// metric is being wired in.
/// </remarks>
public interface IAuditRedactionFailureCounter
{
/// <summary>Increment the audit-redaction failure counter by one.</summary>
void Increment();
}

17
src/ScadaLink.AuditLog/Payload/NoOpAuditRedactionFailureCounter.cs Normal file
View File

@@ -0,0 +1,17 @@
namespace ScadaLink.AuditLog.Payload;
/// <summary>
/// Default <see cref="IAuditRedactionFailureCounter"/> binding used when the
/// Site Health Monitoring bridge has not been wired yet. Bundle C replaces
/// this registration with the real counter that surfaces in the site health
/// report payload as <c>AuditRedactionFailure</c>.
/// </summary>
public sealed class NoOpAuditRedactionFailureCounter : IAuditRedactionFailureCounter
{
/// <inheritdoc/>
public void Increment()
{
// Intentionally empty — Bundle C overrides this binding with the real
// health-metric counter.
}
}

6
src/ScadaLink.AuditLog/ServiceCollectionExtensions.cs
View File

@@ -69,6 +69,12 @@ public static class ServiceCollectionExtensions
// dependency picks up M5-T8 hot reloads on its own.
services.AddSingleton<IAuditPayloadFilter, DefaultAuditPayloadFilter>();
// M5 Bundle B: per-stage redactor-failure counter. NoOp default;
// Bundle C replaces this binding with the Site Health Monitoring
// bridge that surfaces failures as AuditRedactionFailure on the site
// health report.
services.TryAddSingleton<IAuditRedactionFailureCounter, NoOpAuditRedactionFailureCounter>();
// M2 Bundle E: site writer + telemetry options bindings.
// BindConfiguration is not used because the configuration root supplied
// by the caller may not be the application root — we go through the

217
tests/ScadaLink.AuditLog.Tests/Payload/HeaderRedactionTests.cs Normal file
View File

@@ -0,0 +1,217 @@
using System.Text;
using System.Text.Json;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ScadaLink.AuditLog.Configuration;
using ScadaLink.AuditLog.Payload;
using ScadaLink.Commons.Entities.Audit;
using ScadaLink.Commons.Types.Enums;
namespace ScadaLink.AuditLog.Tests.Payload;
/// <summary>
/// Bundle B (M5-T3) tests for <see cref="DefaultAuditPayloadFilter"/> HTTP header
/// redaction. Redaction parses <see cref="AuditEvent.RequestSummary"/> /
/// <see cref="AuditEvent.ResponseSummary"/> as JSON of shape
/// <c>{"headers": {"name": "value", ...}, "body": "..."}</c>, replaces values
/// whose header NAME (case-insensitive) is in
/// <see cref="AuditLogOptions.HeaderRedactList"/> with <c>"&lt;redacted&gt;"</c>,
/// and re-serialises. Non-JSON inputs pass through unchanged (no-op for
/// emitters that have not yet adopted the convention). The stage runs BEFORE
/// truncation so the redaction marker survives the cap.
/// </summary>
public class HeaderRedactionTests
{
private static IOptionsMonitor<AuditLogOptions> Monitor(AuditLogOptions? opts = null) =>
new StaticMonitor(opts ?? new AuditLogOptions());
private static DefaultAuditPayloadFilter Filter(AuditLogOptions? opts = null) =>
new(Monitor(opts), NullLogger<DefaultAuditPayloadFilter>.Instance);
private static AuditEvent NewEvent(
AuditStatus status = AuditStatus.Delivered,
string? request = null,
string? response = null) => new()
{
EventId = Guid.NewGuid(),
OccurredAtUtc = DateTime.UtcNow,
Channel = AuditChannel.ApiOutbound,
Kind = AuditKind.ApiCall,
Status = status,
RequestSummary = request,
ResponseSummary = response,
};
private static string BuildSummary(IDictionary<string, string> headers, string body)
{
// Serialize via System.Text.Json so we get a representative shape.
return JsonSerializer.Serialize(new
{
headers = headers,
body = body,
});
}
private static IDictionary<string, JsonElement> ParseSummary(string? summary)
{
Assert.NotNull(summary);
using var doc = JsonDocument.Parse(summary!);
var dict = new Dictionary<string, JsonElement>();
foreach (var property in doc.RootElement.EnumerateObject())
{
dict[property.Name] = property.Value.Clone();
}
return dict;
}
[Fact]
public void HeaderRedaction_AuthorizationBearer_Redacted()
{
var headers = new Dictionary<string, string>
{
["Authorization"] = "Bearer secret-token-xyz",
["Content-Type"] = "application/json",
};
var input = BuildSummary(headers, "hello");
var evt = NewEvent(request: input);
var result = Filter().Apply(evt);
var parsed = ParseSummary(result.RequestSummary);
var resultHeaders = parsed["headers"];
Assert.Equal("<redacted>", resultHeaders.GetProperty("Authorization").GetString());
}
[Fact]
public void HeaderRedaction_CaseInsensitive_LowercaseAuthorization_Redacted()
{
var headers = new Dictionary<string, string>
{
["authorization"] = "Bearer secret-token-xyz",
};
var input = BuildSummary(headers, "hello");
var evt = NewEvent(request: input);
var result = Filter().Apply(evt);
var parsed = ParseSummary(result.RequestSummary);
var resultHeaders = parsed["headers"];
Assert.Equal("<redacted>", resultHeaders.GetProperty("authorization").GetString());
}
[Fact]
public void HeaderRedaction_CustomRedactList_RedactsCustomHeaderName()
{
var opts = new AuditLogOptions
{
HeaderRedactList = new List<string> { "X-Custom-Secret" },
};
var headers = new Dictionary<string, string>
{
["X-Custom-Secret"] = "topsecret",
["Authorization"] = "Bearer keep-me", // not in list anymore
};
var input = BuildSummary(headers, "hi");
var evt = NewEvent(request: input);
var result = Filter(opts).Apply(evt);
var parsed = ParseSummary(result.RequestSummary);
var resultHeaders = parsed["headers"];
Assert.Equal("<redacted>", resultHeaders.GetProperty("X-Custom-Secret").GetString());
// Authorization no longer listed -> preserved verbatim.
Assert.Equal("Bearer keep-me", resultHeaders.GetProperty("Authorization").GetString());
}
[Fact]
public void HeaderRedaction_NonJson_RequestSummary_Unchanged()
{
const string input = "this is not JSON at all";
var evt = NewEvent(request: input);
var result = Filter().Apply(evt);
Assert.Equal(input, result.RequestSummary);
}
[Fact]
public void HeaderRedaction_NoHeadersField_Unchanged()
{
var input = JsonSerializer.Serialize(new { body = "only a body, no headers" });
var evt = NewEvent(request: input);
var result = Filter().Apply(evt);
// The stage may re-serialise but the content must be semantically identical.
var parsed = ParseSummary(result.RequestSummary);
Assert.Equal("only a body, no headers", parsed["body"].GetString());
Assert.False(parsed.ContainsKey("headers"));
}
[Fact]
public void HeaderRedaction_Other_Headers_Preserved()
{
var headers = new Dictionary<string, string>
{
["Authorization"] = "Bearer secret",
["Content-Type"] = "application/json",
["X-Request-Id"] = "abc-123",
["Accept"] = "application/json",
};
var input = BuildSummary(headers, "payload");
var evt = NewEvent(request: input);
var result = Filter().Apply(evt);
var parsed = ParseSummary(result.RequestSummary);
var resultHeaders = parsed["headers"];
Assert.Equal("<redacted>", resultHeaders.GetProperty("Authorization").GetString());
Assert.Equal("application/json", resultHeaders.GetProperty("Content-Type").GetString());
Assert.Equal("abc-123", resultHeaders.GetProperty("X-Request-Id").GetString());
Assert.Equal("application/json", resultHeaders.GetProperty("Accept").GetString());
}
[Fact]
public void HeaderRedaction_AppliedBeforeTruncation()
{
// Build a summary whose Authorization header value is enormous AND whose
// body padding pushes the total beyond the 8 KB cap. After redaction the
// Authorization value becomes "<redacted>" — then truncation caps the
// re-serialised string. Result must:
// * carry "<redacted>" (header redaction ran first),
// * NOT carry the original secret bytes (proves redaction won, not order swap),
// * be capped at the configured DefaultCapBytes,
// * have PayloadTruncated == true.
const string secret = "SUPER-SECRET-TOKEN-DO-NOT-LEAK";
var headers = new Dictionary<string, string>
{
["Authorization"] = "Bearer " + secret,
};
var body = new string('x', 9 * 1024);
var input = BuildSummary(headers, body);
Assert.True(Encoding.UTF8.GetByteCount(input) > 8192);
var evt = NewEvent(AuditStatus.Delivered, request: input);
var result = Filter().Apply(evt);
Assert.NotNull(result.RequestSummary);
Assert.True(Encoding.UTF8.GetByteCount(result.RequestSummary!) <= 8192);
Assert.Contains("<redacted>", result.RequestSummary);
Assert.DoesNotContain(secret, result.RequestSummary);
Assert.True(result.PayloadTruncated);
}
/// <summary>
/// IOptionsMonitor test double — returns the same snapshot on every read,
/// no change-token plumbing required for these tests.
/// </summary>
private sealed class StaticMonitor : IOptionsMonitor<AuditLogOptions>
{
private readonly AuditLogOptions _value;
public StaticMonitor(AuditLogOptions value) => _value = value;
public AuditLogOptions CurrentValue => _value;
public AuditLogOptions Get(string? name) => _value;
public IDisposable? OnChange(Action<AuditLogOptions, string?> listener) => null;
}
}