using System.Reflection;
using Microsoft.Data.Sqlite;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NSubstitute;
using ZB.MOM.WW.Audit;
using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.TestSupport;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using IAuditWriter = ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services.IAuditWriter;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site;
///
/// Regression coverage.
/// used to call pending.Completion.TrySetResult() per row BEFORE
/// transaction.Commit() ran. A later failure in the SAME batch — another row's insert
/// throwing, or the commit itself failing (SQLITE_FULL, IOERR) — rolled the whole transaction
/// back, but the earlier row's had already completed
/// successfully: the caller, and (whose ring-buffer fallback
/// exists precisely for this outage), both saw false success while the row was actually gone
/// from the database — TrySetException in the outer catch is a no-op against an
/// already-completed .
///
///
/// A committed transaction can't practically be made to fail via public SQLite surface without
/// exhausting disk or injecting a custom VFS, so these tests force a genuine, non-swallowed
/// mid-batch INSERT failure instead (the "bonus defect" the finding also calls out: a mid-batch
/// insert failure on a later row loses the rows that inserted fine before it, the same way).
/// That failure is reached through a real SQLite trigger + a custom scalar function registered
/// directly on the writer's own connection (via reflection —
/// deliberately doesn't expose it), so the whole path under test — transaction, per-row
/// try/catch, rollback, TrySetException — is the writer's real production code, not a mock.
///
public class SqliteAuditWriterCommitFailureTests
{
private const string FailMarker = "P4-2-FORCED-FAILURE";
private static AuditEvent NewEvent(string target) => ScadaBridgeAuditEventFactory.Create(
channel: AuditChannel.ApiOutbound,
kind: AuditKind.ApiCall,
status: AuditStatus.Delivered,
eventId: Guid.NewGuid(),
occurredAtUtc: DateTime.UtcNow,
target: target);
///
/// Installs a BEFORE INSERT trigger on audit_event that calls a custom scalar
/// function throwing whenever the inserted row's Target is .
/// An exception thrown from a Microsoft.Data.Sqlite custom function surfaces as a generic
/// SQLITE_ERROR (code 1), NOT SQLITE_CONSTRAINT (code 19) — so it is NOT caught by
/// FlushBatch's duplicate-EventId swallow (which matches only code 19) and instead reaches
/// the outer catch exactly like a real disk-outage commit failure would.
///
private static void InstallFailureTrigger(SqliteAuditWriter writer)
{
var connectionField = typeof(SqliteAuditWriter).GetField(
"_connection", BindingFlags.NonPublic | BindingFlags.Instance)
?? throw new InvalidOperationException("SqliteAuditWriter._connection not found — test seam broke.");
var connection = (SqliteConnection)connectionField.GetValue(writer)!;
connection.CreateFunction("throw_if_marker", target =>
{
if (target == FailMarker)
{
throw new InvalidOperationException("P4-2 test-forced insert failure");
}
return 0;
});
using var cmd = connection.CreateCommand();
cmd.CommandText = $"""
CREATE TRIGGER test_fail_marker
BEFORE INSERT ON audit_event
FOR EACH ROW WHEN NEW.Target = '{FailMarker}'
BEGIN
SELECT throw_if_marker(NEW.Target);
END;
""";
cmd.ExecuteNonQuery();
}
private static (SqliteAuditWriter writer, string dataSource) CreateWriter(string testName)
{
var dataSource = $"file:{testName}-{Guid.NewGuid():N}?mode=memory&cache=shared";
var opts = new SqliteAuditWriterOptions
{
DatabasePath = dataSource,
// Generous flush window: both writes below are enqueued back-to-back with no
// await between them, so they are already both in the channel long before the
// background loop's first drain attempt — this just gives extra margin so the
// test can't flake into two separate flushes on a slow CI box.
FlushIntervalMs = 1000,
};
var writer = new SqliteAuditWriter(
Options.Create(opts),
NullLogger.Instance,
new FakeNodeIdentityProvider(),
connectionStringOverride: $"Data Source={dataSource};Cache=Shared");
return (writer, dataSource);
}
/// Verifies that a mid-batch insert failure faults every row in the batch, not just the failing row.
/// A task that represents the asynchronous operation.
[Fact]
public async Task FlushBatch_MidBatchInsertFailure_FaultsEveryRowInBatch_NotJustTheFailingRow()
{
var (writer, dataSource) = CreateWriter(
nameof(FlushBatch_MidBatchInsertFailure_FaultsEveryRowInBatch_NotJustTheFailingRow));
await using var _ = writer;
InstallFailureTrigger(writer);
// evt1 is an ordinary row — its own two INSERTs succeed. evt2 carries the marker and
// throws mid-batch, from WITHIN the same transaction as evt1's already-successful
// inserts. Fired without individually awaiting so both land in the channel before the
// background loop's first drain — i.e. the same FlushBatch transaction.
var evt1 = NewEvent("normal-row");
var evt2 = NewEvent(FailMarker);
var t1 = writer.WriteAsync(evt1);
var t2 = writer.WriteAsync(evt2);
try
{
await Task.WhenAll(t1, t2).WaitAsync(TimeSpan.FromSeconds(10));
}
catch
{
// Expected: both tasks fault. Individually asserted below so a failure here
// reports which one didn't behave as expected, not just "something threw".
}
// THE regression assertion: evt1's own inserts succeeded, but the batch it shared
// with evt2 rolled back — its Task must fault, not silently report success. Pre-fix,
// t1 would already be RanToCompletion at this point (TrySetResult ran inside the loop
// before evt2's insert threw), and the later TrySetException in the outer catch would
// be a silent no-op against the already-completed TaskCompletionSource.
Assert.True(t1.IsFaulted,
"evt1's row was rolled back along with the rest of the batch, so its Task must fault instead of reporting false success.");
Assert.True(t2.IsFaulted, "evt2's own insert threw directly and must fault its Task.");
// No commit ever happened for this batch.
Assert.Equal(0, writer.FlushCountForTests);
// And the database genuinely holds neither row — proves the rollback actually
// happened, not just that the Task-completion contract changed.
using var verify = new SqliteConnection($"Data Source={dataSource};Cache=Shared");
verify.Open();
using var countCmd = verify.CreateCommand();
countCmd.CommandText = "SELECT COUNT(*) FROM audit_event;";
Assert.Equal(0L, (long)countCmd.ExecuteScalar()!);
}
/// Verifies that the fallback writer rings every faulted row of a mid-batch insert failure and increments the failure counter once per row.
/// A task that represents the asynchronous operation.
[Fact]
public async Task FallbackAuditWriter_MidBatchInsertFailure_RingsEveryFaultedRow_AndIncrementsFailureCounterForEach()
{
// End-to-end proof the fix actually closes the loop the finding describes:
// FallbackAuditWriter composes the REAL SqliteAuditWriter as its primary. Before the
// fix, evt1's primary write would appear to SUCCEED (its Task completed before the
// rollback), so FallbackAuditWriter would never route it to the ring or count it as a
// failure — the whole batch would be silently lost with SiteAuditWriteFailures staying
// at 0, exactly as the finding describes.
var (writer, _) = CreateWriter(
nameof(FallbackAuditWriter_MidBatchInsertFailure_RingsEveryFaultedRow_AndIncrementsFailureCounterForEach));
await using var _w = writer;
InstallFailureTrigger(writer);
var ring = new RingBufferFallback(capacity: 16);
var counter = Substitute.For();
var fallback = new FallbackAuditWriter(
writer, ring, counter, NullLogger.Instance);
var evt1 = NewEvent("normal-row");
var evt2 = NewEvent(FailMarker);
// FallbackAuditWriter.WriteAsync never throws (audit failures are always swallowed at
// that layer) — fired concurrently, same reasoning as above, so both land in one
// FlushBatch transaction.
var t1 = fallback.WriteAsync(evt1);
var t2 = fallback.WriteAsync(evt2);
await Task.WhenAll(t1, t2).WaitAsync(TimeSpan.FromSeconds(10));
// Both rows — including the one whose own INSERT succeeded before the batch rolled
// back — must have been routed to the ring and counted as a primary failure.
Assert.Equal(2, ring.Count);
counter.Received(2).Increment();
}
}