From 18352442ab93f207acba2986a0c310968490578c Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 10:30:03 -0400 Subject: [PATCH] fix(controlplane): audit writer drops only the rejected row, not the whole batch AuditWriterActor documented per-row dedup on the UX_ConfigAuditLog_EventId index but flushed one statement batch: a single duplicate EventId (any at-least-once producer across a singleton hand-over) threw for all 500 rows (R03:2.6). SaveChanges now detaches the entries a DbUpdateException names (2601/2627 logged at Debug as the expected duplicate, anything else at Warning) and re-saves, bounded by the batch size; a failure naming no entry stays a whole-batch failure. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Audit/AuditWriterActor.cs | 57 ++++++++- .../AuditWriterActorRejectedRowTests.cs | 110 ++++++++++++++++++ 2 files changed, 163 insertions(+), 4 deletions(-) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs index 2ca3026c2..0e20da2fc 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs @@ -17,8 +17,8 @@ namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Audit; /// Dedup is two-layer: in-buffer (the below collapses /// duplicate EventIds before flush) and at the database via the filtered unique index /// UX_ConfigAuditLog_EventId (cross-restart safety — a retry of an already-flushed -/// batch hits the constraint and we drop the duplicate insert without losing the rest of -/// the batch). +/// batch hits the constraint; detaches the rejected row +/// and re-saves, so the duplicate is dropped without losing the rest of the batch). /// /// Implements the shared seam: is a /// best-effort, never-throwing entry point that simply Tells this actor and returns @@ -107,8 +107,8 @@ public sealed class AuditWriterActor : ReceiveActor, IWithTimers, IAuditWriter Outcome = evt.Outcome.ToString(), }); } - db.SaveChanges(); - _log.Debug("AuditWriter flushed {Count} events", snapshot.Count); + var dropped = SaveDetachingRejectedRows(db); + _log.Debug("AuditWriter flushed {Count} events ({Dropped} rejected rows dropped)", snapshot.Count - dropped, dropped); } catch (Exception ex) { @@ -116,6 +116,55 @@ public sealed class AuditWriterActor : ReceiveActor, IWithTimers, IAuditWriter } } + /// + /// Saves the batch, detaching and retrying past any row the database rejects (R03:2.6). One + /// statement batch used to carry the whole flush: a single duplicate EventId (any at-least-once + /// producer — the audit seam is Tell-based through a singleton proxy that re-delivers across a + /// hand-over) threw for all 500 rows and the class remarks' per-row dedup claim was not what ran. + /// + /// The context holding the batch. + /// How many rows were dropped. + /// + /// Only a that NAMES its entries is handled here — that is the shape a + /// constraint violation takes (SQL 2601/2627 for the unique index). A failure with no entries (the + /// connection died, a deadlock) is the whole-batch failure it always was and propagates to the + /// caller's catch. Bounded by the batch size so a store that rejects every row still terminates. + /// + private int SaveDetachingRejectedRows(OtOpcUaConfigDbContext db) + { + var dropped = 0; + var attempts = 0; + var maxAttempts = db.ChangeTracker.Entries().Count() + 1; + while (true) + { + try + { + db.SaveChanges(); + return dropped; + } + catch (DbUpdateException ex) when (ex.Entries.Count > 0 && ++attempts < maxAttempts) + { + var duplicate = IsUniqueViolation(ex); + foreach (var entry in ex.Entries) + { + var eventId = (entry.Entity as ConfigAuditLog)?.EventId; + if (duplicate) + _log.Debug("AuditWriter: EventId {EventId} already persisted (unique index); dropping the duplicate", eventId); + else + _log.Warning(ex, "AuditWriter: the database rejected EventId {EventId}; dropping that row and keeping the batch", eventId); + entry.State = EntityState.Detached; + dropped++; + } + } + } + } + + /// SQL Server 2601 (unique index) / 2627 (unique constraint), the dedup contract's expected rejection. + /// The update failure. + /// for a duplicate-key rejection. + private static bool IsUniqueViolation(DbUpdateException ex) => + ex.InnerException is Microsoft.Data.SqlClient.SqlException { Number: 2601 or 2627 }; + /// protected override void PreRestart(Exception reason, object message) { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs new file mode 100644 index 000000000..bd64161ab --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs @@ -0,0 +1,110 @@ +using Akka.Actor; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage; +using Microsoft.EntityFrameworkCore.Update; +using Shouldly; +using Xunit; +using ZB.MOM.WW.Audit; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Audit; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// R03:2.6 — one constraint hit used to drop the whole flush. The writer now detaches the rejected +/// row(s) and re-saves, so a duplicate EventId costs exactly that row. +/// +public sealed class AuditWriterActorRejectedRowTests : ControlPlaneActorTestBase +{ + [Fact] + public void A_rejected_row_is_dropped_and_the_rest_of_the_batch_is_persisted() + { + var dbName = Guid.NewGuid().ToString("N"); + var root = new InMemoryDatabaseRoot(); + var duplicate = Guid.NewGuid(); + var rejecting = new RejectingDbFactory(dbName, root, new RejectOnce(duplicate)); + var actor = Sys.ActorOf(AuditWriterActor.Props(rejecting)); + + var ids = Enumerable.Range(0, 20).Select(_ => Guid.NewGuid()).ToList(); + ids.Insert(7, duplicate); + foreach (var id in ids) actor.Tell(NewEvent(id)); + + Watch(actor); + actor.Tell(PoisonPill.Instance); // PostStop flushes + ExpectTerminated(actor); + + using var db = rejecting.CreateDbContext(); + var persisted = db.ConfigAuditLogs.Select(r => r.EventId).ToList(); + persisted.Count.ShouldBe(20, "every row but the rejected one lands"); + persisted.ShouldNotContain(duplicate); + // Positive control: the rows on either side of the rejected one are present. + persisted.ShouldContain(ids[6]); + persisted.ShouldContain(ids[8]); + } + + /// A failure that names no entry is a whole-batch failure, exactly as before. + [Fact] + public void A_connection_level_failure_still_drops_the_batch_without_looping() + { + var dbName = Guid.NewGuid().ToString("N"); + var root = new InMemoryDatabaseRoot(); + var rejecting = new RejectingDbFactory(dbName, root, new AlwaysFailWithoutEntries()); + var actor = Sys.ActorOf(AuditWriterActor.Props(rejecting)); + + for (var i = 0; i < 5; i++) actor.Tell(NewEvent(Guid.NewGuid())); + Watch(actor); + actor.Tell(PoisonPill.Instance); + ExpectTerminated(actor); // terminates ⇒ the retry loop did not spin + + using var db = rejecting.CreateDbContext(); + db.ConfigAuditLogs.Count().ShouldBe(0); + } + + private static AuditEvent NewEvent(Guid eventId) => new() + { + EventId = eventId, Category = "Config", Action = "Edit", Actor = "joe", + OccurredAtUtc = DateTimeOffset.UtcNow, Outcome = AuditOutcome.Success, + DetailsJson = "{}", SourceNode = "node-a", CorrelationId = Guid.NewGuid(), + }; + + /// Throws a naming the entry for one EventId, the first time it is saved. + private sealed class RejectOnce(Guid eventId) : ISaveChangesInterceptor + { + private int _fired; + + public InterceptionResult SavingChanges(DbContextEventData eventData, InterceptionResult result) + { + if (_fired == 1 || eventData.Context is null) return result; + var entry = eventData.Context.ChangeTracker.Entries() + .FirstOrDefault(e => e.Entity.EventId == eventId); + if (entry is null) return result; + _fired = 1; + // GetInfrastructure() hands back the pubternal InternalEntityEntry, which is the IUpdateEntry the + // real provider puts on the exception; the test only needs that one cast. +#pragma warning disable EF1001 + var updateEntry = (IUpdateEntry)entry.GetInfrastructure(); +#pragma warning restore EF1001 + throw new DbUpdateException("Cannot insert duplicate key row (simulated 2601)", new[] { updateEntry }); + } + } + + private sealed class AlwaysFailWithoutEntries : ISaveChangesInterceptor + { + public InterceptionResult SavingChanges(DbContextEventData eventData, InterceptionResult result) => + throw new DbUpdateException("connection lost (simulated)"); + } + + private sealed class RejectingDbFactory(string dbName, InMemoryDatabaseRoot root, IInterceptor interceptor) + : IDbContextFactory + { + public OtOpcUaConfigDbContext CreateDbContext() => new( + new DbContextOptionsBuilder() + .UseInMemoryDatabase(dbName, root) + .AddInterceptors(interceptor) + .Options); + } +}