Files
lmxopcua/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs
T
Joseph Doherty 18352442ab fix(controlplane): audit writer drops only the rejected row, not the whole batch
AuditWriterActor documented per-row dedup on the UX_ConfigAuditLog_EventId index but flushed one
statement batch: a single duplicate EventId (any at-least-once producer across a singleton
hand-over) threw for all 500 rows (R03:2.6). SaveChanges now detaches the entries a
DbUpdateException names (2601/2627 logged at Debug as the expected duplicate, anything else at
Warning) and re-saves, bounded by the batch size; a failure naming no entry stays a whole-batch
failure.

Refs #733

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
2026-10-22 10:30:03 -04:00

188 lines
8.2 KiB
C#

using Akka.Actor;
using Akka.Event;
using Microsoft.EntityFrameworkCore;
using ZB.MOM.WW.Audit;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Audit;
/// <summary>
/// Cluster-singleton actor that batches <see cref="AuditEvent"/> messages from the cluster
/// and bulk-inserts them into <c>ConfigAuditLog</c>. Flush triggers:
/// - Buffer reaches <see cref="FlushBatchSize"/> events.
/// - <see cref="FlushInterval"/> elapses with a non-empty buffer.
/// - <c>PreRestart</c> / <c>PostStop</c> (supervisor swap or coordinated shutdown).
///
/// Dedup is two-layer: in-buffer (the <see cref="Dictionary{TKey, TValue}"/> below collapses
/// duplicate EventIds before flush) and at the database via the filtered unique index
/// <c>UX_ConfigAuditLog_EventId</c> (cross-restart safety — a retry of an already-flushed
/// batch hits the constraint; <see cref="SaveDetachingRejectedRows"/> detaches the rejected row
/// and re-saves, so the duplicate is dropped without losing the rest of the batch).
///
/// Implements the shared <see cref="IAuditWriter"/> seam: <see cref="WriteAsync"/> is a
/// best-effort, never-throwing entry point that simply <c>Tell</c>s this actor and returns
/// a completed task, so non-Akka callers can emit canonical audit events through the same
/// batching/dedup pipeline as in-cluster <c>Tell</c> traffic.
/// </summary>
public sealed class AuditWriterActor : ReceiveActor, IWithTimers, IAuditWriter
{
public const int FlushBatchSize = 500;
public static readonly TimeSpan FlushInterval = TimeSpan.FromSeconds(5);
private readonly IDbContextFactory<OtOpcUaConfigDbContext> _dbFactory;
private readonly ILoggingAdapter _log = Context.GetLogger();
private readonly Dictionary<Guid, AuditEvent> _buffer = new();
/// <summary>Gets or sets the timer scheduler for the actor.</summary>
public ITimerScheduler Timers { get; set; } = null!;
/// <summary>Creates a Props factory for the AuditWriterActor.</summary>
/// <param name="dbFactory">The database context factory for creating ConfigDb connections.</param>
/// <returns>The <see cref="Akka.Actor.Props"/> used to create an <see cref="AuditWriterActor"/>.</returns>
public static Props Props(IDbContextFactory<OtOpcUaConfigDbContext> dbFactory) =>
Akka.Actor.Props.Create(() => new AuditWriterActor(dbFactory));
/// <summary>Initializes a new instance of the AuditWriterActor class.</summary>
/// <param name="dbFactory">The database context factory for creating ConfigDb connections.</param>
public AuditWriterActor(IDbContextFactory<OtOpcUaConfigDbContext> dbFactory)
{
_dbFactory = dbFactory;
Receive<AuditEvent>(HandleEvent);
Receive<Flush>(_ => FlushBuffer());
}
/// <inheritdoc />
protected override void PreStart()
{
Timers.StartPeriodicTimer("flush", Flush.Instance, FlushInterval);
}
/// <summary>
/// <see cref="IAuditWriter"/> seam. Best-effort and never throws: routes the event onto this
/// actor's mailbox via <c>Tell</c> (thread-safe from any caller) so it flows through the same
/// batching + dedup pipeline as in-cluster traffic, then returns immediately. The actual
/// persistence happens asynchronously on the next flush; a write failure there is logged and
/// the batch dropped (per the best-effort audit contract).
/// </summary>
/// <param name="evt">The canonical audit event to persist.</param>
/// <param name="ct">Unused — enqueue is synchronous and non-blocking.</param>
/// <returns>A completed task.</returns>
public Task WriteAsync(AuditEvent evt, CancellationToken ct = default)
{
// Akka Tell is safe to call from any thread and never throws to the caller.
Self.Tell(evt);
return Task.CompletedTask;
}
private void HandleEvent(AuditEvent evt)
{
// In-buffer dedup. Last write wins on duplicate EventId within the batch — events
// with the same EventId are by contract identical, so this is a no-op.
_buffer[evt.EventId] = evt;
if (_buffer.Count >= FlushBatchSize) FlushBuffer();
}
private void FlushBuffer()
{
if (_buffer.Count == 0) return;
var snapshot = _buffer.Values.ToList();
_buffer.Clear();
try
{
using var db = _dbFactory.CreateDbContext();
foreach (var evt in snapshot)
{
db.ConfigAuditLogs.Add(new ConfigAuditLog
{
Timestamp = evt.OccurredAtUtc.UtcDateTime,
Principal = evt.Actor,
EventType = $"{evt.Category}:{evt.Action}",
NodeId = evt.SourceNode,
DetailsJson = evt.DetailsJson,
EventId = evt.EventId,
CorrelationId = evt.CorrelationId,
Outcome = evt.Outcome.ToString(),
});
}
var dropped = SaveDetachingRejectedRows(db);
_log.Debug("AuditWriter flushed {Count} events ({Dropped} rejected rows dropped)", snapshot.Count - dropped, dropped);
}
catch (Exception ex)
{
_log.Error(ex, "AuditWriter flush failed; {Count} events dropped", snapshot.Count);
}
}
/// <summary>
/// Saves the batch, detaching and retrying past any row the database rejects (R03:2.6). One
/// statement batch used to carry the whole flush: a single duplicate <c>EventId</c> (any at-least-once
/// producer — the audit seam is Tell-based through a singleton proxy that re-delivers across a
/// hand-over) threw for all 500 rows and the class remarks' per-row dedup claim was not what ran.
/// </summary>
/// <param name="db">The context holding the batch.</param>
/// <returns>How many rows were dropped.</returns>
/// <remarks>
/// Only a <see cref="DbUpdateException"/> that NAMES its entries is handled here — that is the shape a
/// constraint violation takes (SQL 2601/2627 for the unique index). A failure with no entries (the
/// connection died, a deadlock) is the whole-batch failure it always was and propagates to the
/// caller's catch. Bounded by the batch size so a store that rejects every row still terminates.
/// </remarks>
private int SaveDetachingRejectedRows(OtOpcUaConfigDbContext db)
{
var dropped = 0;
var attempts = 0;
var maxAttempts = db.ChangeTracker.Entries<ConfigAuditLog>().Count() + 1;
while (true)
{
try
{
db.SaveChanges();
return dropped;
}
catch (DbUpdateException ex) when (ex.Entries.Count > 0 && ++attempts < maxAttempts)
{
var duplicate = IsUniqueViolation(ex);
foreach (var entry in ex.Entries)
{
var eventId = (entry.Entity as ConfigAuditLog)?.EventId;
if (duplicate)
_log.Debug("AuditWriter: EventId {EventId} already persisted (unique index); dropping the duplicate", eventId);
else
_log.Warning(ex, "AuditWriter: the database rejected EventId {EventId}; dropping that row and keeping the batch", eventId);
entry.State = EntityState.Detached;
dropped++;
}
}
}
}
/// <summary>SQL Server 2601 (unique index) / 2627 (unique constraint), the dedup contract's expected rejection.</summary>
/// <param name="ex">The update failure.</param>
/// <returns><see langword="true"/> for a duplicate-key rejection.</returns>
private static bool IsUniqueViolation(DbUpdateException ex) =>
ex.InnerException is Microsoft.Data.SqlClient.SqlException { Number: 2601 or 2627 };
/// <inheritdoc />
protected override void PreRestart(Exception reason, object message)
{
FlushBuffer();
base.PreRestart(reason, message);
}
/// <inheritdoc />
protected override void PostStop()
{
FlushBuffer();
base.PostStop();
}
public sealed class Flush
{
public static readonly Flush Instance = new();
private Flush() { }
}
}