fix(auditlog): explicit permanent rejections on the ingest ack (IngestAck.rejected_event_ids, field 2) with a dedicated counter; reconciler ignores transient faults for abandonment; persist the served high-water mark; prune only on a non-empty enumeration (review 2026-10-21 06/PERF-1, 06/IMPL-1 adversarial follow-up)
This commit is contained in:
+32
-8
@@ -18,7 +18,6 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site;
|
||||
public class SqliteAuditWriterForwardStateTests : IDisposable
|
||||
{
|
||||
private readonly string _dbPath = Path.Combine(Path.GetTempPath(), $"audit-fwdstate-{Guid.NewGuid():N}.db");
|
||||
private readonly CountingFailureCounter _counter = new();
|
||||
|
||||
/// <summary>Deletes the temporary database.</summary>
|
||||
public void Dispose()
|
||||
@@ -32,8 +31,7 @@ public class SqliteAuditWriterForwardStateTests : IDisposable
|
||||
private SqliteAuditWriter CreateWriter() => new(
|
||||
Options.Create(new SqliteAuditWriterOptions { DatabasePath = _dbPath }),
|
||||
NullLogger<SqliteAuditWriter>.Instance,
|
||||
new FakeNodeIdentityProvider(),
|
||||
failureCounter: _counter);
|
||||
new FakeNodeIdentityProvider());
|
||||
|
||||
private static async Task<Guid> WriteAsync(SqliteAuditWriter writer, DateTime occurredAtUtc)
|
||||
{
|
||||
@@ -132,13 +130,39 @@ public class SqliteAuditWriterForwardStateTests : IDisposable
|
||||
Assert.Equal(behind, Assert.Single(await writer.ReadPendingAsync(1)).EventId);
|
||||
Assert.Contains(await writer.ReadPendingSinceAsync(DateTime.MinValue, 100), e => e.EventId == poison);
|
||||
Assert.Equal(0, await writer.PurgeExpiredAsync(DateTime.UtcNow.AddDays(1)));
|
||||
Assert.Equal(1, _counter.Count);
|
||||
Assert.Equal(1, writer.RejectedPushRowCount);
|
||||
}
|
||||
|
||||
private sealed class CountingFailureCounter : IAuditWriteFailureCounter
|
||||
/// <summary>
|
||||
/// The served high-water mark survives a writer restart: rows served before the restart are
|
||||
/// retired by the first cursor that covers them after it, instead of being stranded owed
|
||||
/// (and counted on the backlog) until something happens to re-serve them.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task ServedHighWaterMark_SurvivesRestart_SoTheNextCursorStillRetires()
|
||||
{
|
||||
private int _count;
|
||||
public int Count => Volatile.Read(ref _count);
|
||||
public void Increment() => Interlocked.Increment(ref _count);
|
||||
var t = DateTime.UtcNow.AddMinutes(-5);
|
||||
Guid abandoned;
|
||||
IReadOnlyList<AuditEvent> served;
|
||||
await using (var writer = CreateWriter())
|
||||
{
|
||||
abandoned = await WriteAsync(writer, t);
|
||||
await writer.MarkPushAbandonedAsync(new[] { abandoned });
|
||||
served = await writer.ReadPendingSinceAsync(DateTime.MinValue, 100);
|
||||
Assert.Single(served);
|
||||
}
|
||||
|
||||
await using (var restarted = CreateWriter())
|
||||
{
|
||||
Assert.True(restarted.MaxServedRowIdForTests > 0);
|
||||
|
||||
var flipped = await restarted.MarkReconciledUpToAsync(
|
||||
served[0].OccurredAtUtc.UtcDateTime, served[0].EventId.ToString());
|
||||
|
||||
Assert.Equal(1, flipped);
|
||||
Assert.Equal("Reconciled", restarted.GetForwardStateForTests(abandoned)!.Value.State);
|
||||
Assert.Equal(0, (await restarted.GetBacklogStatsAsync()).PendingCount);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+18
-13
@@ -785,27 +785,33 @@ public class SiteAuditTelemetryActorTests : TestKit
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Review 2026-10-21 06/IMPL-1: rows central ANSWERED for but did not accept get a refused
|
||||
/// attempt recorded (so a poison head row eventually leaves the push queue); accepted rows
|
||||
/// do not, and a transport fault records nothing.
|
||||
/// Review 2026-10-21 06/IMPL-1 (adversarial follow-up): only rows central EXPLICITLY lists in
|
||||
/// <c>RejectedEventIds</c> get a refused attempt recorded. A row merely missing from the accepted
|
||||
/// list (central DB outage / budget exhaustion) and a transport fault record nothing.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Drain_RefusedRows_RecordRejectedAttempts_TransportFaultDoesNot()
|
||||
public async Task Drain_OnlyExplicitlyRejectedRows_RecordRefusals()
|
||||
{
|
||||
var queue = Substitute.For<ISiteAuditQueue, ZB.MOM.WW.ScadaBridge.AuditLog.Site.ISiteAuditPushLifecycle>();
|
||||
var lifecycle = (ZB.MOM.WW.ScadaBridge.AuditLog.Site.ISiteAuditPushLifecycle)queue;
|
||||
var accepted = NewEvent();
|
||||
var refused = NewEvent();
|
||||
var poison = NewEvent();
|
||||
var unanswered = NewEvent();
|
||||
var rows = new[] { accepted, poison, unanswered };
|
||||
queue.ReadPendingAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
|
||||
.Returns(
|
||||
Task.FromResult<IReadOnlyList<AuditEvent>>(new[] { accepted, refused }),
|
||||
Task.FromResult<IReadOnlyList<AuditEvent>>(new[] { accepted, refused }),
|
||||
Task.FromResult<IReadOnlyList<AuditEvent>>(rows),
|
||||
Task.FromResult<IReadOnlyList<AuditEvent>>(rows),
|
||||
Task.FromResult<IReadOnlyList<AuditEvent>>(rows),
|
||||
Task.FromResult<IReadOnlyList<AuditEvent>>(Array.Empty<AuditEvent>()));
|
||||
var explicitAck = AckAll(new[] { accepted });
|
||||
explicitAck.RejectedEventIds.Add(poison.EventId.ToString());
|
||||
_client.IngestAuditEventsAsync(Arg.Any<AuditEventBatch>(), Arg.Any<CancellationToken>())
|
||||
.Returns(
|
||||
_ => Task.FromException<IngestAck>(new InvalidOperationException("transport down")),
|
||||
_ => Task.FromResult(AckAll(new[] { accepted })));
|
||||
_ => Task.FromResult(AckAll(new[] { accepted })), // partial ack, no verdict (old central / DB outage)
|
||||
_ => Task.FromResult(explicitAck));
|
||||
|
||||
Sys.ActorOf(Props.Create(() => new SiteAuditTelemetryActor(
|
||||
queue, _client, Opts(), NullLogger<SiteAuditTelemetryActor>.Instance,
|
||||
@@ -813,13 +819,12 @@ public class SiteAuditTelemetryActorTests : TestKit
|
||||
|
||||
await AwaitAssertAsync(async () =>
|
||||
await lifecycle.Received(1).RecordRejectedPushAttemptsAsync(
|
||||
Arg.Is<IReadOnlyList<Guid>>(g => g.Count == 1 && g[0] == refused.EventId),
|
||||
Arg.Is<IReadOnlyList<Guid>>(g => g.Count == 1 && g[0] == poison.EventId),
|
||||
10,
|
||||
Arg.Any<CancellationToken>()),
|
||||
TimeSpan.FromSeconds(10));
|
||||
await queue.Received(1).MarkForwardedAsync(
|
||||
Arg.Is<IReadOnlyList<Guid>>(g => g.Count == 1 && g[0] == accepted.EventId),
|
||||
Arg.Any<CancellationToken>());
|
||||
TimeSpan.FromSeconds(15));
|
||||
await lifecycle.Received(1).RecordRejectedPushAttemptsAsync(
|
||||
Arg.Any<IReadOnlyList<Guid>>(), Arg.Any<int>(), Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
private sealed class CapturingLogger<T> : Microsoft.Extensions.Logging.ILogger<T>
|
||||
|
||||
Reference in New Issue
Block a user