fix(auditlog): explicit permanent rejections on the ingest ack (IngestAck.rejected_event_ids, field 2) with a dedicated counter; reconciler ignores transient faults for abandonment; persist the served high-water mark; prune only on a non-empty enumeration (review 2026-10-21 06/PERF-1, 06/IMPL-1 adversarial follow-up)

This commit is contained in:
Joseph Doherty
2026-10-22 08:40:42 -04:00
parent 38167bca88
commit b0a08bb63b
23 changed files with 790 additions and 168 deletions
@@ -785,27 +785,33 @@ public class SiteAuditTelemetryActorTests : TestKit
}
/// <summary>
/// Review 2026-10-21 06/IMPL-1: rows central ANSWERED for but did not accept get a refused
/// attempt recorded (so a poison head row eventually leaves the push queue); accepted rows
/// do not, and a transport fault records nothing.
/// Review 2026-10-21 06/IMPL-1 (adversarial follow-up): only rows central EXPLICITLY lists in
/// <c>RejectedEventIds</c> get a refused attempt recorded. A row merely missing from the accepted
/// list (central DB outage / budget exhaustion) and a transport fault record nothing.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Drain_RefusedRows_RecordRejectedAttempts_TransportFaultDoesNot()
public async Task Drain_OnlyExplicitlyRejectedRows_RecordRefusals()
{
var queue = Substitute.For<ISiteAuditQueue, ZB.MOM.WW.ScadaBridge.AuditLog.Site.ISiteAuditPushLifecycle>();
var lifecycle = (ZB.MOM.WW.ScadaBridge.AuditLog.Site.ISiteAuditPushLifecycle)queue;
var accepted = NewEvent();
var refused = NewEvent();
var poison = NewEvent();
var unanswered = NewEvent();
var rows = new[] { accepted, poison, unanswered };
queue.ReadPendingAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(
Task.FromResult<IReadOnlyList<AuditEvent>>(new[] { accepted, refused }),
Task.FromResult<IReadOnlyList<AuditEvent>>(new[] { accepted, refused }),
Task.FromResult<IReadOnlyList<AuditEvent>>(rows),
Task.FromResult<IReadOnlyList<AuditEvent>>(rows),
Task.FromResult<IReadOnlyList<AuditEvent>>(rows),
Task.FromResult<IReadOnlyList<AuditEvent>>(Array.Empty<AuditEvent>()));
var explicitAck = AckAll(new[] { accepted });
explicitAck.RejectedEventIds.Add(poison.EventId.ToString());
_client.IngestAuditEventsAsync(Arg.Any<AuditEventBatch>(), Arg.Any<CancellationToken>())
.Returns(
_ => Task.FromException<IngestAck>(new InvalidOperationException("transport down")),
_ => Task.FromResult(AckAll(new[] { accepted })));
_ => Task.FromResult(AckAll(new[] { accepted })), // partial ack, no verdict (old central / DB outage)
_ => Task.FromResult(explicitAck));
Sys.ActorOf(Props.Create(() => new SiteAuditTelemetryActor(
queue, _client, Opts(), NullLogger<SiteAuditTelemetryActor>.Instance,
@@ -813,13 +819,12 @@ public class SiteAuditTelemetryActorTests : TestKit
await AwaitAssertAsync(async () =>
await lifecycle.Received(1).RecordRejectedPushAttemptsAsync(
Arg.Is<IReadOnlyList<Guid>>(g => g.Count == 1 && g[0] == refused.EventId),
Arg.Is<IReadOnlyList<Guid>>(g => g.Count == 1 && g[0] == poison.EventId),
10,
Arg.Any<CancellationToken>()),
TimeSpan.FromSeconds(10));
await queue.Received(1).MarkForwardedAsync(
Arg.Is<IReadOnlyList<Guid>>(g => g.Count == 1 && g[0] == accepted.EventId),
Arg.Any<CancellationToken>());
TimeSpan.FromSeconds(15));
await lifecycle.Received(1).RecordRejectedPushAttemptsAsync(
Arg.Any<IReadOnlyList<Guid>>(), Arg.Any<int>(), Arg.Any<CancellationToken>());
}
private sealed class CapturingLogger<T> : Microsoft.Extensions.Logging.ILogger<T>