dohertj2/mxaccessgw
1
0
Fork 0
Code Issues 3 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix(audit): write-through on completed channel, poison-batch isolation, drain-fault fallback

Browse Source
This commit is contained in:
Joseph Doherty
2026-08-15 12:37:09 -04:00
parent 07b83561d1
commit 7b2d04605e
6 changed files with 492 additions and 274 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.MxGateway.Tests/Security/Audit/ChannelAuditWriterTests.cs
+132 -23
View File
@@ -1,3 +1,4 @@
using Microsoft.Data.Sqlite;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Time.Testing;
using ZB.MOM.WW.Audit;
@@ -10,10 +11,11 @@ namespace ZB.MOM.WW.MxGateway.Tests.Security.Audit;
/// <summary>
/// Tests the bounded, asynchronous audit path: <see cref="ChannelAuditWriter"/> (enqueue-only
/// once a drain is attached, drops rather than blocks when the channel is full) and
/// <see cref="AuditDrainService"/> (batched drain, one-time table bootstrap, retention sweep).
/// The channel makes the already-documented best-effort audit contract explicit and bounded:
/// a partially denied bulk RPC no longer pays a SQLite round-trip per denied tag.
/// once a drain is attached, drops rather than blocks when the channel is full, writes through
/// whenever nothing is draining) and <see cref="AuditDrainService"/> (batched drain, poison-batch
/// isolation, one-time table bootstrap, retention sweep). The channel makes the already-documented
/// best-effort audit contract explicit and bounded: a partially denied bulk RPC no longer pays a
/// SQLite round-trip per denied tag.
/// </summary>
public sealed class ChannelAuditWriterTests : IDisposable
{
@@ -71,6 +73,29 @@ public sealed class ChannelAuditWriterTests : IDisposable
Assert.All(sink.BatchSizes, size => Assert.True(size <= AuditDrainService.MaxBatchSize));
}
/// <summary>
/// A batch that will not commit is retried one event at a time, so a single unwritable row
/// costs only itself instead of taking the up-to-63 good events sharing its transaction.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task DrainPendingAsync_WhenBatchFails_RetriesIndividuallyAndKeepsSurvivors()
{
CountingAuditSink sink = new() { PoisonAction = "poison" };
(ChannelAuditWriter writer, AuditDrainService drain) = CreateWriterAndDrain(sink);
writer.AttachDrain();
await writer.WriteAsync(MakeEvent("good-1"), CancellationToken.None);
await writer.WriteAsync(MakeEvent("poison"), CancellationToken.None);
await writer.WriteAsync(MakeEvent("good-2"), CancellationToken.None);
int persisted = await drain.DrainPendingAsync(CancellationToken.None);
// The batch commit fails on the poison event; the per-event retry still lands both others.
Assert.Equal(2, persisted);
Assert.Equal(["good-1", "good-2"], sink.Events.Select(auditEvent => auditEvent.Action));
}
/// <summary>
/// When the bounded channel is full the write is dropped rather than blocking the caller,
/// and the drop is counted. A stalled or slow audit database must never stall an RPC.
@@ -89,13 +114,12 @@ public sealed class ChannelAuditWriterTests : IDisposable
// Every call must complete synchronously: the channel never blocks a producer.
Task write = writer.WriteAsync(MakeEvent($"denied-{index}"), CancellationToken.None);
Assert.True(write.IsCompletedSuccessfully);
await write;
}
Assert.Equal(overflow, writer.DroppedCount);
// The channel still holds exactly its capacity; nothing reached the sink (no drain ran).
Assert.Equal(0, sink.InsertBatchCalls);
await Task.CompletedTask;
}
/// <summary>
@@ -114,6 +138,26 @@ public sealed class ChannelAuditWriterTests : IDisposable
Assert.Equal("dashboard-create-key", Assert.Single(sink.Events).Action);
}
/// <summary>
/// A completed channel has no future reader, so an attached writer must write through rather
/// than discard. This is the re-attach-after-shutdown footgun: enqueueing would lose 100% of
/// audit silently.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task WriteAsync_AfterChannelCompleted_WritesThroughInsteadOfDiscarding()
{
CountingAuditSink sink = new();
(ChannelAuditWriter writer, _) = CreateWriterAndDrain(sink);
writer.AttachDrain();
writer.CompleteWriting();
await writer.WriteAsync(MakeEvent("constraint-denied"), CancellationToken.None);
Assert.Equal("constraint-denied", Assert.Single(sink.Events).Action);
Assert.Equal(0, writer.DroppedCount);
}
/// <summary>
/// The retention sweep deletes rows older than <c>AuditRetentionDays</c>, measured from the
/// injected clock, and runs once at startup after the one-time table bootstrap.
@@ -137,29 +181,42 @@ public sealed class ChannelAuditWriterTests : IDisposable
}
/// <summary>
/// The real SQLite sink deletes only rows older than the cutoff, and its bootstrap latch
/// means the <c>CREATE TABLE</c> round-trip is not repeated on every subsequent write.
/// The sweep compares instants, not stored text, and never deletes a row it cannot date.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task SqliteStore_DeleteOlderThan_RemovesOnlyRowsBeyondTheRetentionWindow()
public async Task SqliteStore_DeleteOlderThan_ComparesInstantsAndKeepsUnparseableRows()
{
SqliteCanonicalAuditStore store = CreateStore();
DateTimeOffset now = new(2026, 8, 15, 12, 0, 0, TimeSpan.Zero);
(SqliteCanonicalAuditStore store, AuthSqliteConnectionFactory factory) = CreateStore();
DateTimeOffset cutoff = new(2026, 5, 17, 12, 0, 0, TimeSpan.Zero);
await store.InsertBatchAsync(
[
MakeEvent("stale", now.AddDays(-120)),
// A non-UTC offset must still compare correctly against the cutoff.
MakeEvent("stale-offset", now.AddDays(-100).ToOffset(TimeSpan.FromHours(-5))),
MakeEvent("fresh", now.AddDays(-1)),
MakeEvent("swept", new DateTimeOffset(2026, 5, 17, 11, 0, 0, TimeSpan.Zero)),
MakeEvent("kept-fresh", new DateTimeOffset(2026, 5, 18, 0, 0, 0, TimeSpan.Zero)),
],
CancellationToken.None);
int deleted = await store.DeleteOlderThanAsync(now.AddDays(-90), CancellationToken.None);
// Both discriminating rows are written as raw text, because AuditEvent.OccurredAtUtc
// normalizes to UTC and so cannot express them. They stand for audit that reached the
// table any other way — a repair script, an older schema, a future producer.
//
// 09:00 at -05:00 is 14:00 UTC, two hours AFTER the cutoff, yet its text sorts BEFORE
// "2026-05-17T12:00:00.0000000+00:00". A lexicographic sweep deletes it; comparing
// instants keeps it.
await InsertRawRowAsync(factory, "kept-offset", "2026-05-17T09:00:00.1234567-05:00");
Assert.Equal(2, deleted);
Assert.Equal("fresh", Assert.Single(await store.ListRecentAsync(10, CancellationToken.None)).Action);
// Undateable audit is kept, never guessed at. This text sorts BELOW the cutoff, so a
// lexicographic sweep deletes it, while datetime() yields NULL and leaves it alone.
await InsertRawRowAsync(factory, "kept-unparseable", "0000-not-a-timestamp");
int deleted = await store.DeleteOlderThanAsync(cutoff, CancellationToken.None);
Assert.Equal(1, deleted);
Assert.Equal(
["kept-fresh", "kept-offset", "kept-unparseable"],
(await ListActionsAsync(factory)).OrderBy(action => action, StringComparer.Ordinal));
}
private static (ChannelAuditWriter Writer, AuditDrainService Drain) CreateWriterAndDrain(
@@ -179,11 +236,51 @@ public sealed class ChannelAuditWriterTests : IDisposable
return (writer, drain);
}
private SqliteCanonicalAuditStore CreateStore()
private (SqliteCanonicalAuditStore Store, AuthSqliteConnectionFactory Factory) CreateStore()
{
TempDatabaseDirectory directory = TempDatabaseDirectory.Create("mxgateway-channel-audit");
_tempDirectories.Add(directory);
return new SqliteCanonicalAuditStore(new AuthSqliteConnectionFactory(directory.DatabasePath()));
AuthSqliteConnectionFactory factory = new(directory.DatabasePath());
return (new SqliteCanonicalAuditStore(factory), factory);
}
// Writes an audit row whose occurred_at_utc bypasses the store's DateTimeOffset formatting,
// so the sweep can be shown to leave undateable audit alone.
private static async Task InsertRawRowAsync(
AuthSqliteConnectionFactory factory,
string action,
string occurredAtUtc)
{
await using SqliteConnection connection = await factory.OpenConnectionAsync(CancellationToken.None);
await using SqliteCommand command = connection.CreateCommand();
command.CommandText =
"""
INSERT INTO audit_event (event_id, occurred_at_utc, actor, action, outcome)
VALUES ($event_id, $occurred_at_utc, 'operator01', $action, 'Denied');
""";
command.Parameters.AddWithValue("$event_id", Guid.NewGuid().ToString());
command.Parameters.AddWithValue("$occurred_at_utc", occurredAtUtc);
command.Parameters.AddWithValue("$action", action);
await command.ExecuteNonQueryAsync(CancellationToken.None);
}
// Reads actions straight from SQL: ListRecentAsync would throw on the unparseable timestamp.
private static async Task<List<string>> ListActionsAsync(AuthSqliteConnectionFactory factory)
{
await using SqliteConnection connection = await factory.OpenConnectionAsync(CancellationToken.None);
await using SqliteCommand command = connection.CreateCommand();
command.CommandText = "SELECT action FROM audit_event;";
List<string> actions = [];
await using SqliteDataReader reader = await command.ExecuteReaderAsync(CancellationToken.None);
while (await reader.ReadAsync(CancellationToken.None))
{
actions.Add(reader.GetString(0));
}
return actions;
}
private static AuditEvent MakeEvent(string action, DateTimeOffset? occurredAtUtc = null) => new()
@@ -209,22 +306,27 @@ public sealed class ChannelAuditWriterTests : IDisposable
/// <summary>
/// In-memory <see cref="IAuditEventSink"/> that records every call, so a test can prove the
/// write path did NOT touch the sink and that the drain batched what it did write.
/// write path did NOT touch the sink and that the drain batched what it did write. Setting
/// <see cref="PoisonAction"/> makes any write containing that action fail, modelling a row
/// the store refuses.
/// </summary>
private sealed class CountingAuditSink : IAuditEventSink
{
private readonly Lock _gate = new();
/// <summary>Gets the action whose presence makes a write fail, or null to accept everything.</summary>
public string? PoisonAction { get; init; }
/// <summary>Gets the events handed to the sink, in the order they were written.</summary>
public List<AuditEvent> Events { get; } = [];
/// <summary>Gets the size of each batch the sink was asked to insert.</summary>
/// <summary>Gets the size of each batch the sink accepted.</summary>
public List<int> BatchSizes { get; } = [];
/// <summary>Gets the cutoffs the sink was asked to delete below.</summary>
public List<DateTimeOffset> DeleteCutoffs { get; } = [];
/// <summary>Gets the number of <see cref="InsertBatchAsync"/> calls.</summary>
/// <summary>Gets the number of accepted <see cref="InsertBatchAsync"/> calls.</summary>
public int InsertBatchCalls => BatchSizes.Count;
/// <summary>Gets the number of <see cref="EnsureInitializedAsync"/> calls.</summary>
@@ -248,6 +350,13 @@ public sealed class ChannelAuditWriterTests : IDisposable
/// <inheritdoc />
public Task InsertBatchAsync(IReadOnlyList<AuditEvent> auditEvents, CancellationToken cancellationToken)
{
if (PoisonAction is not null
&& auditEvents.Any(auditEvent => auditEvent.Action == PoisonAction))
{
return Task.FromException(
new InvalidOperationException($"Refused a write of {auditEvents.Count} audit events."));
}
lock (_gate)
{
BatchSizes.Add(auditEvents.Count);