fix(controlplane): audit writer drops only the rejected row, not the whole batch

AuditWriterActor documented per-row dedup on the UX_ConfigAuditLog_EventId index but flushed one
statement batch: a single duplicate EventId (any at-least-once producer across a singleton
hand-over) threw for all 500 rows (R03:2.6). SaveChanges now detaches the entries a
DbUpdateException names (2601/2627 logged at Debug as the expected duplicate, anything else at
Warning) and re-saves, bounded by the batch size; a failure naming no entry stays a whole-batch
failure.

Refs #733

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
2026-10-22 10:30:03 -04:00
parent 1bf5ef916c
commit 18352442ab
2 changed files with 163 additions and 4 deletions
@@ -17,8 +17,8 @@ namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Audit;
/// Dedup is two-layer: in-buffer (the <see cref="Dictionary{TKey, TValue}"/> below collapses
/// duplicate EventIds before flush) and at the database via the filtered unique index
/// <c>UX_ConfigAuditLog_EventId</c> (cross-restart safety — a retry of an already-flushed
/// batch hits the constraint and we drop the duplicate insert without losing the rest of
/// the batch).
/// batch hits the constraint; <see cref="SaveDetachingRejectedRows"/> detaches the rejected row
/// and re-saves, so the duplicate is dropped without losing the rest of the batch).
///
/// Implements the shared <see cref="IAuditWriter"/> seam: <see cref="WriteAsync"/> is a
/// best-effort, never-throwing entry point that simply <c>Tell</c>s this actor and returns
@@ -107,8 +107,8 @@ public sealed class AuditWriterActor : ReceiveActor, IWithTimers, IAuditWriter
Outcome = evt.Outcome.ToString(),
});
}
db.SaveChanges();
_log.Debug("AuditWriter flushed {Count} events", snapshot.Count);
var dropped = SaveDetachingRejectedRows(db);
_log.Debug("AuditWriter flushed {Count} events ({Dropped} rejected rows dropped)", snapshot.Count - dropped, dropped);
}
catch (Exception ex)
{
@@ -116,6 +116,55 @@ public sealed class AuditWriterActor : ReceiveActor, IWithTimers, IAuditWriter
}
}
/// <summary>
/// Saves the batch, detaching and retrying past any row the database rejects (R03:2.6). One
/// statement batch used to carry the whole flush: a single duplicate <c>EventId</c> (any at-least-once
/// producer — the audit seam is Tell-based through a singleton proxy that re-delivers across a
/// hand-over) threw for all 500 rows and the class remarks' per-row dedup claim was not what ran.
/// </summary>
/// <param name="db">The context holding the batch.</param>
/// <returns>How many rows were dropped.</returns>
/// <remarks>
/// Only a <see cref="DbUpdateException"/> that NAMES its entries is handled here — that is the shape a
/// constraint violation takes (SQL 2601/2627 for the unique index). A failure with no entries (the
/// connection died, a deadlock) is the whole-batch failure it always was and propagates to the
/// caller's catch. Bounded by the batch size so a store that rejects every row still terminates.
/// </remarks>
private int SaveDetachingRejectedRows(OtOpcUaConfigDbContext db)
{
var dropped = 0;
var attempts = 0;
var maxAttempts = db.ChangeTracker.Entries<ConfigAuditLog>().Count() + 1;
while (true)
{
try
{
db.SaveChanges();
return dropped;
}
catch (DbUpdateException ex) when (ex.Entries.Count > 0 && ++attempts < maxAttempts)
{
var duplicate = IsUniqueViolation(ex);
foreach (var entry in ex.Entries)
{
var eventId = (entry.Entity as ConfigAuditLog)?.EventId;
if (duplicate)
_log.Debug("AuditWriter: EventId {EventId} already persisted (unique index); dropping the duplicate", eventId);
else
_log.Warning(ex, "AuditWriter: the database rejected EventId {EventId}; dropping that row and keeping the batch", eventId);
entry.State = EntityState.Detached;
dropped++;
}
}
}
}
/// <summary>SQL Server 2601 (unique index) / 2627 (unique constraint), the dedup contract's expected rejection.</summary>
/// <param name="ex">The update failure.</param>
/// <returns><see langword="true"/> for a duplicate-key rejection.</returns>
private static bool IsUniqueViolation(DbUpdateException ex) =>
ex.InnerException is Microsoft.Data.SqlClient.SqlException { Number: 2601 or 2627 };
/// <inheritdoc />
protected override void PreRestart(Exception reason, object message)
{
@@ -0,0 +1,110 @@
using Akka.Actor;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Diagnostics;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Storage;
using Microsoft.EntityFrameworkCore.Update;
using Shouldly;
using Xunit;
using ZB.MOM.WW.Audit;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Audit;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
/// <summary>
/// R03:2.6 — one constraint hit used to drop the whole flush. The writer now detaches the rejected
/// row(s) and re-saves, so a duplicate <c>EventId</c> costs exactly that row.
/// </summary>
public sealed class AuditWriterActorRejectedRowTests : ControlPlaneActorTestBase
{
[Fact]
public void A_rejected_row_is_dropped_and_the_rest_of_the_batch_is_persisted()
{
var dbName = Guid.NewGuid().ToString("N");
var root = new InMemoryDatabaseRoot();
var duplicate = Guid.NewGuid();
var rejecting = new RejectingDbFactory(dbName, root, new RejectOnce(duplicate));
var actor = Sys.ActorOf(AuditWriterActor.Props(rejecting));
var ids = Enumerable.Range(0, 20).Select(_ => Guid.NewGuid()).ToList();
ids.Insert(7, duplicate);
foreach (var id in ids) actor.Tell(NewEvent(id));
Watch(actor);
actor.Tell(PoisonPill.Instance); // PostStop flushes
ExpectTerminated(actor);
using var db = rejecting.CreateDbContext();
var persisted = db.ConfigAuditLogs.Select(r => r.EventId).ToList();
persisted.Count.ShouldBe(20, "every row but the rejected one lands");
persisted.ShouldNotContain(duplicate);
// Positive control: the rows on either side of the rejected one are present.
persisted.ShouldContain(ids[6]);
persisted.ShouldContain(ids[8]);
}
/// <summary>A failure that names no entry is a whole-batch failure, exactly as before.</summary>
[Fact]
public void A_connection_level_failure_still_drops_the_batch_without_looping()
{
var dbName = Guid.NewGuid().ToString("N");
var root = new InMemoryDatabaseRoot();
var rejecting = new RejectingDbFactory(dbName, root, new AlwaysFailWithoutEntries());
var actor = Sys.ActorOf(AuditWriterActor.Props(rejecting));
for (var i = 0; i < 5; i++) actor.Tell(NewEvent(Guid.NewGuid()));
Watch(actor);
actor.Tell(PoisonPill.Instance);
ExpectTerminated(actor); // terminates ⇒ the retry loop did not spin
using var db = rejecting.CreateDbContext();
db.ConfigAuditLogs.Count().ShouldBe(0);
}
private static AuditEvent NewEvent(Guid eventId) => new()
{
EventId = eventId, Category = "Config", Action = "Edit", Actor = "joe",
OccurredAtUtc = DateTimeOffset.UtcNow, Outcome = AuditOutcome.Success,
DetailsJson = "{}", SourceNode = "node-a", CorrelationId = Guid.NewGuid(),
};
/// <summary>Throws a <see cref="DbUpdateException"/> naming the entry for one EventId, the first time it is saved.</summary>
private sealed class RejectOnce(Guid eventId) : ISaveChangesInterceptor
{
private int _fired;
public InterceptionResult<int> SavingChanges(DbContextEventData eventData, InterceptionResult<int> result)
{
if (_fired == 1 || eventData.Context is null) return result;
var entry = eventData.Context.ChangeTracker.Entries<ConfigAuditLog>()
.FirstOrDefault(e => e.Entity.EventId == eventId);
if (entry is null) return result;
_fired = 1;
// GetInfrastructure() hands back the pubternal InternalEntityEntry, which is the IUpdateEntry the
// real provider puts on the exception; the test only needs that one cast.
#pragma warning disable EF1001
var updateEntry = (IUpdateEntry)entry.GetInfrastructure();
#pragma warning restore EF1001
throw new DbUpdateException("Cannot insert duplicate key row (simulated 2601)", new[] { updateEntry });
}
}
private sealed class AlwaysFailWithoutEntries : ISaveChangesInterceptor
{
public InterceptionResult<int> SavingChanges(DbContextEventData eventData, InterceptionResult<int> result) =>
throw new DbUpdateException("connection lost (simulated)");
}
private sealed class RejectingDbFactory(string dbName, InMemoryDatabaseRoot root, IInterceptor interceptor)
: IDbContextFactory<OtOpcUaConfigDbContext>
{
public OtOpcUaConfigDbContext CreateDbContext() => new(
new DbContextOptionsBuilder<OtOpcUaConfigDbContext>()
.UseInMemoryDatabase(dbName, root)
.AddInterceptors(interceptor)
.Options);
}
}