fix(auditlog): explicit permanent rejections on the ingest ack (IngestAck.rejected_event_ids, field 2) with a dedicated counter; reconciler ignores transient faults for abandonment; persist the served high-water mark; prune only on a non-empty enumeration (review 2026-10-21 06/PERF-1, 06/IMPL-1 adversarial follow-up)

This commit is contained in:
Joseph Doherty
2026-10-22 08:40:42 -04:00
parent 38167bca88
commit b0a08bb63b
23 changed files with 790 additions and 168 deletions
@@ -18,7 +18,6 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site;
public class SqliteAuditWriterForwardStateTests : IDisposable
{
private readonly string _dbPath = Path.Combine(Path.GetTempPath(), $"audit-fwdstate-{Guid.NewGuid():N}.db");
private readonly CountingFailureCounter _counter = new();
/// <summary>Deletes the temporary database.</summary>
public void Dispose()
@@ -32,8 +31,7 @@ public class SqliteAuditWriterForwardStateTests : IDisposable
private SqliteAuditWriter CreateWriter() => new(
Options.Create(new SqliteAuditWriterOptions { DatabasePath = _dbPath }),
NullLogger<SqliteAuditWriter>.Instance,
new FakeNodeIdentityProvider(),
failureCounter: _counter);
new FakeNodeIdentityProvider());
private static async Task<Guid> WriteAsync(SqliteAuditWriter writer, DateTime occurredAtUtc)
{
@@ -132,13 +130,39 @@ public class SqliteAuditWriterForwardStateTests : IDisposable
Assert.Equal(behind, Assert.Single(await writer.ReadPendingAsync(1)).EventId);
Assert.Contains(await writer.ReadPendingSinceAsync(DateTime.MinValue, 100), e => e.EventId == poison);
Assert.Equal(0, await writer.PurgeExpiredAsync(DateTime.UtcNow.AddDays(1)));
Assert.Equal(1, _counter.Count);
Assert.Equal(1, writer.RejectedPushRowCount);
}
private sealed class CountingFailureCounter : IAuditWriteFailureCounter
/// <summary>
/// The served high-water mark survives a writer restart: rows served before the restart are
/// retired by the first cursor that covers them after it, instead of being stranded owed
/// (and counted on the backlog) until something happens to re-serve them.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task ServedHighWaterMark_SurvivesRestart_SoTheNextCursorStillRetires()
{
private int _count;
public int Count => Volatile.Read(ref _count);
public void Increment() => Interlocked.Increment(ref _count);
var t = DateTime.UtcNow.AddMinutes(-5);
Guid abandoned;
IReadOnlyList<AuditEvent> served;
await using (var writer = CreateWriter())
{
abandoned = await WriteAsync(writer, t);
await writer.MarkPushAbandonedAsync(new[] { abandoned });
served = await writer.ReadPendingSinceAsync(DateTime.MinValue, 100);
Assert.Single(served);
}
await using (var restarted = CreateWriter())
{
Assert.True(restarted.MaxServedRowIdForTests > 0);
var flipped = await restarted.MarkReconciledUpToAsync(
served[0].OccurredAtUtc.UtcDateTime, served[0].EventId.ToString());
Assert.Equal(1, flipped);
Assert.Equal("Reconciled", restarted.GetForwardStateForTests(abandoned)!.Value.State);
Assert.Equal(0, (await restarted.GetBacklogStatsAsync()).PendingCount);
}
}
}
@@ -785,27 +785,33 @@ public class SiteAuditTelemetryActorTests : TestKit
}
/// <summary>
/// Review 2026-10-21 06/IMPL-1: rows central ANSWERED for but did not accept get a refused
/// attempt recorded (so a poison head row eventually leaves the push queue); accepted rows
/// do not, and a transport fault records nothing.
/// Review 2026-10-21 06/IMPL-1 (adversarial follow-up): only rows central EXPLICITLY lists in
/// <c>RejectedEventIds</c> get a refused attempt recorded. A row merely missing from the accepted
/// list (central DB outage / budget exhaustion) and a transport fault record nothing.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Drain_RefusedRows_RecordRejectedAttempts_TransportFaultDoesNot()
public async Task Drain_OnlyExplicitlyRejectedRows_RecordRefusals()
{
var queue = Substitute.For<ISiteAuditQueue, ZB.MOM.WW.ScadaBridge.AuditLog.Site.ISiteAuditPushLifecycle>();
var lifecycle = (ZB.MOM.WW.ScadaBridge.AuditLog.Site.ISiteAuditPushLifecycle)queue;
var accepted = NewEvent();
var refused = NewEvent();
var poison = NewEvent();
var unanswered = NewEvent();
var rows = new[] { accepted, poison, unanswered };
queue.ReadPendingAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(
Task.FromResult<IReadOnlyList<AuditEvent>>(new[] { accepted, refused }),
Task.FromResult<IReadOnlyList<AuditEvent>>(new[] { accepted, refused }),
Task.FromResult<IReadOnlyList<AuditEvent>>(rows),
Task.FromResult<IReadOnlyList<AuditEvent>>(rows),
Task.FromResult<IReadOnlyList<AuditEvent>>(rows),
Task.FromResult<IReadOnlyList<AuditEvent>>(Array.Empty<AuditEvent>()));
var explicitAck = AckAll(new[] { accepted });
explicitAck.RejectedEventIds.Add(poison.EventId.ToString());
_client.IngestAuditEventsAsync(Arg.Any<AuditEventBatch>(), Arg.Any<CancellationToken>())
.Returns(
_ => Task.FromException<IngestAck>(new InvalidOperationException("transport down")),
_ => Task.FromResult(AckAll(new[] { accepted })));
_ => Task.FromResult(AckAll(new[] { accepted })), // partial ack, no verdict (old central / DB outage)
_ => Task.FromResult(explicitAck));
Sys.ActorOf(Props.Create(() => new SiteAuditTelemetryActor(
queue, _client, Opts(), NullLogger<SiteAuditTelemetryActor>.Instance,
@@ -813,13 +819,12 @@ public class SiteAuditTelemetryActorTests : TestKit
await AwaitAssertAsync(async () =>
await lifecycle.Received(1).RecordRejectedPushAttemptsAsync(
Arg.Is<IReadOnlyList<Guid>>(g => g.Count == 1 && g[0] == refused.EventId),
Arg.Is<IReadOnlyList<Guid>>(g => g.Count == 1 && g[0] == poison.EventId),
10,
Arg.Any<CancellationToken>()),
TimeSpan.FromSeconds(10));
await queue.Received(1).MarkForwardedAsync(
Arg.Is<IReadOnlyList<Guid>>(g => g.Count == 1 && g[0] == accepted.EventId),
Arg.Any<CancellationToken>());
TimeSpan.FromSeconds(15));
await lifecycle.Received(1).RecordRejectedPushAttemptsAsync(
Arg.Any<IReadOnlyList<Guid>>(), Arg.Any<int>(), Arg.Any<CancellationToken>());
}
private sealed class CapturingLogger<T> : Microsoft.Extensions.Logging.ILogger<T>