fix(auditlog): reconciliation abandons only row-intrinsic failures and only when a sibling row inserted (review 2026-10-21 follow-up M1)

This commit is contained in:
Joseph Doherty
2026-10-22 09:31:26 -04:00
parent 6b8a91cc83
commit 1f87942ce3
4 changed files with 225 additions and 73 deletions
@@ -5,6 +5,7 @@ using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.TestSupport;
using ZB.MOM.WW.Audit;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Entities;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
@@ -254,13 +255,29 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
/// <summary>Gets the audit events recorded by this stub.</summary>
public List<AuditEvent> Inserted { get; } = new();
private readonly HashSet<Guid> _seen = new();
private readonly Guid _poisonId;
private readonly Guid _healthyId;
/// <summary>Initializes the stub.</summary>
/// <param name="poisonId">The row that fails with a row-intrinsic (permanent) SQL error.</param>
/// <param name="healthyId">The one non-synthetic row that inserts; every other row fails transiently.</param>
public AlwaysFailUnlessSyntheticRepo(Guid poisonId, Guid healthyId)
{
_poisonId = poisonId;
_healthyId = healthyId;
}
/// <inheritdoc />
public Task InsertIfNotExistsAsync(AuditEvent evt, CancellationToken ct = default)
{
if (!evt.Action.EndsWith(".ReconciliationAbandoned", StringComparison.Ordinal))
if (evt.EventId == _poisonId)
{
throw new InvalidOperationException("simulated permanent insert failure");
throw SqlExceptionFactory.Create(8152);
}
if (evt.EventId != _healthyId
&& !evt.Action.EndsWith(".ReconciliationAbandoned", StringComparison.Ordinal))
{
throw new TimeoutException("simulated transient insert failure");
}
if (_seen.Add(evt.EventId))
{
@@ -900,12 +917,15 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
// Five pulls each returning the same poison row so its per-EventId attempt
// counter reaches MaxPermanentInsertAttempts (5); afterwards the scripted
// client returns empty and the actor settles.
// A healthy sibling (earlier timestamp) rides every page: it proves the database
// accepts writes, which is what lets the poison row be abandoned at all (follow-up M1).
// The poison fault is a ROW-intrinsic SQL error (8152 string truncation).
var healthy = NewEvent(siteId, poisonTime.AddSeconds(-1));
var responses = Enumerable.Range(0, 5)
.Select(_ => new PullAuditEventsResponse(new[] { poison }, MoreAvailable: false))
.Select(_ => new PullAuditEventsResponse(new[] { healthy, poison }, MoreAvailable: false))
.ToArray();
var client = new ScriptedPullClient().Script(siteId, responses);
var repo = new PoisonInsertRepo(
poison.EventId, new InvalidOperationException("poison row will never insert"));
var repo = new PoisonInsertRepo(poison.EventId, SqlExceptionFactory.Create(8152));
var actor = CreateActor(sites, client, repo, FastTickOptions());
@@ -965,6 +985,9 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
var siteId = "siteReabandon";
var poisonTime = new DateTime(2026, 5, 20, 10, 5, 0, DateTimeKind.Utc);
var poison = NewEvent(siteId, poisonTime);
// A row that always inserts rides every page so the poison row is abandonable
// (an all-rows-failing page is a database problem and never abandons — follow-up M1).
var healthy = NewEvent(siteId, poisonTime.AddSeconds(-1));
// 10 scripted responses: enough for exactly two 5-attempt abandon
// cycles of the poison row (the cursor never advances between them,
@@ -974,12 +997,12 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
// row count is bounded and deterministic.
var responses = Enumerable.Range(0, 10)
.Select(i => new PullAuditEventsResponse(
new[] { poison, NewEvent(siteId, poisonTime.AddSeconds(i + 1)) },
new[] { healthy, poison, NewEvent(siteId, poisonTime.AddSeconds(i + 1)) },
MoreAvailable: false))
.ToArray();
var client = new ScriptedPullClient().Script(siteId, responses);
var sites = new StaticEnumerator(new SiteEntry(siteId, "http://x:8083"));
var repo = new AlwaysFailUnlessSyntheticRepo();
var repo = new AlwaysFailUnlessSyntheticRepo(poison.EventId, healthy.EventId);
var actor = CreateActor(sites, client, repo, FastTickOptions());