fix(auditlog): P5-6 — deterministic synthetic ReconciliationAbandoned EventId
SiteAuditReconciliationActor minted a fresh random EventId for the synthetic ReconciliationAbandoned row every time a permanently-abandoned row was re-abandoned. Because the reconciliation cursor is held at `since` by ANY co-batch unresolved row (see PullSiteAsync's hasUnresolvedFailure gate), a poisoned row's per-EventId attempt counter restarts every time it reappears in a stuck batch, so it gets re-abandoned roughly every MaxPermanentInsertAttempts ticks — minting an unbounded number of duplicate central audit rows for the SAME lost event. Fix: DeriveAbandonmentEventId(Guid lostEventId) derives the synthetic EventId deterministically (SHA-256 over a fixed namespace GUID + the lost row's bytes, RFC4122-v5-shaped). Re-abandonment of the same lost row now always produces the same synthetic EventId, so IAuditLogRepository.InsertIfNotExistsAsync's existing idempotency collapses every repeat into the first insert. Also adds an in-memory _abandonedEventIds set as a CHEAP SHORT-CIRCUIT ONLY (skips the wasted ~25 min retry-then-abandon cycle for a row already known abandoned this process's lifetime) — it is not the correctness mechanism; it resets on singleton restart/failover, at which point the deterministic id alone still keeps re-abandonment idempotent. Tests (SiteAuditReconciliationActorTests): - ReAbandonment_SameLostEventId_YieldsExactlyOneCentralRow: end-to-end repro of the defect trigger (poison row + a rotating-EventId co-batch row that never itself reaches the threshold, holding the cursor forever) across two full abandon cycles. Proof of failure: run against the unfixed source (git stash of the source file only, pure-function tests temporarily #if false'd to keep the file compiling), the test failed with exactly the predicted defect shape — 2 distinct synthetic rows for one lost EventId. Passes post-fix (1 row). - DeriveAbandonmentEventId_DifferentLostRows_YieldDistinctSyntheticIds and DeriveAbandonmentEventId_IsPureAndStableAcrossIndependentInvocations: pure-function coverage of the derivation directly, including a pinned golden-value regression guard proving it is a function of the input GUID's value alone (no clock/randomness/process state).
This commit is contained in:
Joseph Doherty
+182
@@ -176,6 +176,69 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
|
||||
Task.FromResult<IReadOnlyList<string>>(Array.Empty<string>());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Recording stub whose <see cref="InsertIfNotExistsAsync"/> throws for EVERY
|
||||
/// row except a synthetic <see cref="AuditKind.ReconciliationAbandoned"/> row
|
||||
/// (identified by its <c>Action</c> suffix, mirroring <see cref="PoisonInsertRepo"/>'s
|
||||
/// assertion pattern). Used to model a site whose backlog contains one row that
|
||||
/// NEVER inserts (the eventual permanent-abandon target) alongside a co-batch
|
||||
/// row that also never inserts but changes EventId every pull — i.e. it never
|
||||
/// individually reaches <c>MaxPermanentInsertAttempts</c>, so it keeps
|
||||
/// <c>hasUnresolvedFailure</c> true and holds the cursor at <c>since</c> forever
|
||||
/// (P5-6's defect trigger). Dedupes inserted rows on <see cref="AuditEvent.EventId"/>
|
||||
/// exactly like the real <c>InsertIfNotExistsAsync</c>, so a test asserting on
|
||||
/// <see cref="Inserted"/>'s count is exercising the same idempotency contract
|
||||
/// production relies on.
|
||||
/// </summary>
|
||||
private sealed class AlwaysFailUnlessSyntheticRepo : IAuditLogRepository
|
||||
{
|
||||
public List<AuditEvent> Inserted { get; } = new();
|
||||
private readonly HashSet<Guid> _seen = new();
|
||||
|
||||
public Task InsertIfNotExistsAsync(AuditEvent evt, CancellationToken ct = default)
|
||||
{
|
||||
if (!evt.Action.EndsWith(".ReconciliationAbandoned", StringComparison.Ordinal))
|
||||
{
|
||||
throw new InvalidOperationException("simulated permanent insert failure");
|
||||
}
|
||||
if (_seen.Add(evt.EventId))
|
||||
{
|
||||
Inserted.Add(evt);
|
||||
}
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<AuditEvent>> QueryAsync(
|
||||
AuditLogQueryFilter filter, AuditLogPaging paging, CancellationToken ct = default) =>
|
||||
Task.FromResult<IReadOnlyList<AuditEvent>>(Inserted);
|
||||
|
||||
public Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default) =>
|
||||
Task.FromResult(0L);
|
||||
|
||||
public Task<long> PurgeChannelOlderThanAsync(
|
||||
string channel, DateTime threshold, int batchSize, TimeSpan? commandTimeout = null, CancellationToken ct = default) =>
|
||||
Task.FromResult(0L);
|
||||
|
||||
public Task<long> BackfillSourceNodeAsync(
|
||||
string sentinel, DateTime before, int batchSize, CancellationToken ct = default) =>
|
||||
Task.FromResult(0L);
|
||||
|
||||
public Task<IReadOnlyList<DateTime>> GetPartitionBoundariesOlderThanAsync(
|
||||
DateTime threshold, CancellationToken ct = default) =>
|
||||
Task.FromResult<IReadOnlyList<DateTime>>(Array.Empty<DateTime>());
|
||||
|
||||
public Task<ZB.MOM.WW.ScadaBridge.Commons.Types.AuditLogKpiSnapshot> GetKpiSnapshotAsync(
|
||||
TimeSpan window, DateTime? nowUtc = null, CancellationToken ct = default) =>
|
||||
Task.FromResult(new ZB.MOM.WW.ScadaBridge.Commons.Types.AuditLogKpiSnapshot(0L, 0L, 0L, nowUtc ?? DateTime.UtcNow));
|
||||
|
||||
public Task<IReadOnlyList<ExecutionTreeNode>> GetExecutionTreeAsync(
|
||||
Guid executionId, CancellationToken ct = default) =>
|
||||
Task.FromResult<IReadOnlyList<ExecutionTreeNode>>(Array.Empty<ExecutionTreeNode>());
|
||||
|
||||
public Task<IReadOnlyList<string>> GetDistinctSourceNodesAsync(CancellationToken ct = default) =>
|
||||
Task.FromResult<IReadOnlyList<string>>(Array.Empty<string>());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// In-memory enumerator returning a static list of sites.
|
||||
/// </summary>
|
||||
@@ -727,4 +790,123 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
|
||||
|
||||
StopReconciliationActor(actor);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// 10. P5-6: deterministic synthetic ReconciliationAbandoned EventId
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Reproduces the P5-6 defect end to end: a permanently-poisoned row sits
|
||||
/// in a batch alongside a co-batch row that ALSO fails on every pull but
|
||||
/// carries a fresh <see cref="AuditEvent.EventId"/> each time (modelling a
|
||||
/// site that keeps producing new events with transient insert failures).
|
||||
/// Per <c>PullSiteAsync</c>'s <c>hasUnresolvedFailure</c> rule the rotating
|
||||
/// row never individually reaches <c>MaxPermanentInsertAttempts</c>, so it
|
||||
/// holds the reconciliation cursor at <c>since</c> forever — the poison row
|
||||
/// keeps being re-pulled with a reset attempt counter (its
|
||||
/// <c>_failedInsertAttempts</c> entry was cleared by its first abandonment)
|
||||
/// and is permanently abandoned again roughly every
|
||||
/// <c>MaxPermanentInsertAttempts</c> ticks. Ten ticks reproduces two full
|
||||
/// abandon cycles for the SAME poisoned EventId. Pre-fix, each cycle mints a
|
||||
/// fresh random synthetic <c>EventId</c>, so two DISTINCT rows land in the
|
||||
/// repository. Post-fix, <c>DeriveAbandonmentEventId</c> derives the same id
|
||||
/// both times and <c>InsertIfNotExistsAsync</c>'s idempotency collapses the
|
||||
/// second insert into a no-op, leaving exactly one row.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void ReAbandonment_SameLostEventId_YieldsExactlyOneCentralRow()
|
||||
{
|
||||
var siteId = "siteReabandon";
|
||||
var poisonTime = new DateTime(2026, 5, 20, 10, 5, 0, DateTimeKind.Utc);
|
||||
var poison = NewEvent(siteId, poisonTime);
|
||||
|
||||
// 10 scripted responses: enough for exactly two 5-attempt abandon
|
||||
// cycles of the poison row (the cursor never advances between them,
|
||||
// so the SAME poison row + a freshly-guid'd rotating sibling are
|
||||
// re-served every tick). Once exhausted, ScriptedPullClient returns
|
||||
// empty responses, so no further cycles occur — the test's eventual
|
||||
// row count is bounded and deterministic.
|
||||
var responses = Enumerable.Range(0, 10)
|
||||
.Select(i => new PullAuditEventsResponse(
|
||||
new[] { poison, NewEvent(siteId, poisonTime.AddSeconds(i + 1)) },
|
||||
MoreAvailable: false))
|
||||
.ToArray();
|
||||
var client = new ScriptedPullClient().Script(siteId, responses);
|
||||
var sites = new StaticEnumerator(new SiteEntry(siteId, "http://x:8083"));
|
||||
var repo = new AlwaysFailUnlessSyntheticRepo();
|
||||
|
||||
var actor = CreateActor(sites, client, repo, FastTickOptions());
|
||||
|
||||
// Wait for all 10 scripted pulls to have been consumed, i.e. both
|
||||
// abandon cycles have run to completion.
|
||||
AwaitAssert(
|
||||
() => Assert.True(client.Calls.Count >= 10, $"expected >= 10 pull calls, got {client.Calls.Count}"),
|
||||
duration: TimeSpan.FromSeconds(10),
|
||||
interval: TimeSpan.FromMilliseconds(100));
|
||||
|
||||
// Give the actor's mailbox a chance to fully drain the 10th tick's
|
||||
// processing (the call count above only proves PullAsync was invoked,
|
||||
// not that the resulting insert(s) have landed yet) before reading
|
||||
// the repository state.
|
||||
AwaitAssert(
|
||||
() => Assert.True(
|
||||
repo.Inserted.Any(e => e.Action.EndsWith(".ReconciliationAbandoned", StringComparison.Ordinal)),
|
||||
"expected at least one synthetic ReconciliationAbandoned row"),
|
||||
duration: TimeSpan.FromSeconds(5),
|
||||
interval: TimeSpan.FromMilliseconds(100));
|
||||
|
||||
StopReconciliationActor(actor);
|
||||
|
||||
var synthetic = repo.Inserted
|
||||
.Where(e => e.Action.EndsWith(".ReconciliationAbandoned", StringComparison.Ordinal))
|
||||
.ToList();
|
||||
|
||||
// The defect: pre-fix this is 2 (a fresh random EventId per abandon
|
||||
// cycle defeats InsertIfNotExistsAsync's dedup). Post-fix it is 1.
|
||||
Assert.Single(synthetic);
|
||||
Assert.Contains(poison.EventId.ToString(), synthetic[0].DetailsJson);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pure-function coverage of <see cref="SiteAuditReconciliationActor.DeriveAbandonmentEventId"/>
|
||||
/// directly: distinct lost rows must derive distinct synthetic ids (so two
|
||||
/// different permanently-abandoned rows don't collide into one central row).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void DeriveAbandonmentEventId_DifferentLostRows_YieldDistinctSyntheticIds()
|
||||
{
|
||||
var lostA = Guid.NewGuid();
|
||||
var lostB = Guid.NewGuid();
|
||||
|
||||
var derivedA = SiteAuditReconciliationActor.DeriveAbandonmentEventId(lostA);
|
||||
var derivedB = SiteAuditReconciliationActor.DeriveAbandonmentEventId(lostB);
|
||||
|
||||
Assert.NotEqual(derivedA, derivedB);
|
||||
Assert.NotEqual(lostA, derivedA);
|
||||
Assert.NotEqual(lostB, derivedB);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pure-function coverage proving the derivation depends ONLY on the lost
|
||||
/// row's <see cref="AuditEvent.EventId"/> VALUE — not on object identity, not
|
||||
/// on any clock/random/process state. Two independently-parsed <see cref="Guid"/>
|
||||
/// values that are equal by value must derive the identical synthetic id, and
|
||||
/// the exact value is pinned as a regression guard: since the function has no
|
||||
/// external input besides <paramref name="lostEventId"/>, this literal must
|
||||
/// reproduce identically in any process — this run, CI, or a reconciliation
|
||||
/// singleton after a failover restart — which is the "stable across process
|
||||
/// restarts" contract the fix depends on for idempotent re-abandonment.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void DeriveAbandonmentEventId_IsPureAndStableAcrossIndependentInvocations()
|
||||
{
|
||||
var lostEventId = Guid.Parse("6b2f1e3a-8c9d-4a2b-9e3f-1a2b3c4d5e6f");
|
||||
var roundTripped = Guid.Parse(lostEventId.ToString("D"));
|
||||
|
||||
var first = SiteAuditReconciliationActor.DeriveAbandonmentEventId(lostEventId);
|
||||
var second = SiteAuditReconciliationActor.DeriveAbandonmentEventId(roundTripped);
|
||||
|
||||
Assert.Equal(first, second);
|
||||
Assert.Equal(Guid.Parse("36baac53-f4c6-835e-9a77-d0b350a4e0d9"), first);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user