using System.Reflection; using Microsoft.Data.Sqlite; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using NSubstitute; using ZB.MOM.WW.Audit; using ZB.MOM.WW.ScadaBridge.AuditLog.Site; using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.TestSupport; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services; using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; using IAuditWriter = ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services.IAuditWriter; namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site; /// /// Regression coverage. /// used to call pending.Completion.TrySetResult() per row BEFORE /// transaction.Commit() ran. A later failure in the SAME batch — another row's insert /// throwing, or the commit itself failing (SQLITE_FULL, IOERR) — rolled the whole transaction /// back, but the earlier row's had already completed /// successfully: the caller, and (whose ring-buffer fallback /// exists precisely for this outage), both saw false success while the row was actually gone /// from the database — TrySetException in the outer catch is a no-op against an /// already-completed . /// /// /// A committed transaction can't practically be made to fail via public SQLite surface without /// exhausting disk or injecting a custom VFS, so these tests force a genuine, non-swallowed /// mid-batch INSERT failure instead (the "bonus defect" the finding also calls out: a mid-batch /// insert failure on a later row loses the rows that inserted fine before it, the same way). /// That failure is reached through a real SQLite trigger + a custom scalar function registered /// directly on the writer's own connection (via reflection — /// deliberately doesn't expose it), so the whole path under test — transaction, per-row /// try/catch, rollback, TrySetException — is the writer's real production code, not a mock. /// public class SqliteAuditWriterCommitFailureTests { private const string FailMarker = "P4-2-FORCED-FAILURE"; private static AuditEvent NewEvent(string target) => ScadaBridgeAuditEventFactory.Create( channel: AuditChannel.ApiOutbound, kind: AuditKind.ApiCall, status: AuditStatus.Delivered, eventId: Guid.NewGuid(), occurredAtUtc: DateTime.UtcNow, target: target); /// /// Installs a BEFORE INSERT trigger on audit_event that calls a custom scalar /// function throwing whenever the inserted row's Target is . /// An exception thrown from a Microsoft.Data.Sqlite custom function surfaces as a generic /// SQLITE_ERROR (code 1), NOT SQLITE_CONSTRAINT (code 19) — so it is NOT caught by /// FlushBatch's duplicate-EventId swallow (which matches only code 19) and instead reaches /// the outer catch exactly like a real disk-outage commit failure would. /// private static void InstallFailureTrigger(SqliteAuditWriter writer) { var connectionField = typeof(SqliteAuditWriter).GetField( "_connection", BindingFlags.NonPublic | BindingFlags.Instance) ?? throw new InvalidOperationException("SqliteAuditWriter._connection not found — test seam broke."); var connection = (SqliteConnection)connectionField.GetValue(writer)!; connection.CreateFunction("throw_if_marker", target => { if (target == FailMarker) { throw new InvalidOperationException("P4-2 test-forced insert failure"); } return 0; }); using var cmd = connection.CreateCommand(); cmd.CommandText = $""" CREATE TRIGGER test_fail_marker BEFORE INSERT ON audit_event FOR EACH ROW WHEN NEW.Target = '{FailMarker}' BEGIN SELECT throw_if_marker(NEW.Target); END; """; cmd.ExecuteNonQuery(); } private static (SqliteAuditWriter writer, string dataSource) CreateWriter(string testName) { var dataSource = $"file:{testName}-{Guid.NewGuid():N}?mode=memory&cache=shared"; var opts = new SqliteAuditWriterOptions { DatabasePath = dataSource, // Generous flush window: both writes below are enqueued back-to-back with no // await between them, so they are already both in the channel long before the // background loop's first drain attempt — this just gives extra margin so the // test can't flake into two separate flushes on a slow CI box. FlushIntervalMs = 1000, }; var writer = new SqliteAuditWriter( Options.Create(opts), NullLogger.Instance, new FakeNodeIdentityProvider(), connectionStringOverride: $"Data Source={dataSource};Cache=Shared"); return (writer, dataSource); } /// Verifies that a mid-batch insert failure faults every row in the batch, not just the failing row. /// A task that represents the asynchronous operation. [Fact] public async Task FlushBatch_MidBatchInsertFailure_FaultsEveryRowInBatch_NotJustTheFailingRow() { var (writer, dataSource) = CreateWriter( nameof(FlushBatch_MidBatchInsertFailure_FaultsEveryRowInBatch_NotJustTheFailingRow)); await using var _ = writer; InstallFailureTrigger(writer); // evt1 is an ordinary row — its own two INSERTs succeed. evt2 carries the marker and // throws mid-batch, from WITHIN the same transaction as evt1's already-successful // inserts. Fired without individually awaiting so both land in the channel before the // background loop's first drain — i.e. the same FlushBatch transaction. var evt1 = NewEvent("normal-row"); var evt2 = NewEvent(FailMarker); var t1 = writer.WriteAsync(evt1); var t2 = writer.WriteAsync(evt2); try { await Task.WhenAll(t1, t2).WaitAsync(TimeSpan.FromSeconds(10)); } catch { // Expected: both tasks fault. Individually asserted below so a failure here // reports which one didn't behave as expected, not just "something threw". } // THE regression assertion: evt1's own inserts succeeded, but the batch it shared // with evt2 rolled back — its Task must fault, not silently report success. Pre-fix, // t1 would already be RanToCompletion at this point (TrySetResult ran inside the loop // before evt2's insert threw), and the later TrySetException in the outer catch would // be a silent no-op against the already-completed TaskCompletionSource. Assert.True(t1.IsFaulted, "evt1's row was rolled back along with the rest of the batch, so its Task must fault instead of reporting false success."); Assert.True(t2.IsFaulted, "evt2's own insert threw directly and must fault its Task."); // No commit ever happened for this batch. Assert.Equal(0, writer.FlushCountForTests); // And the database genuinely holds neither row — proves the rollback actually // happened, not just that the Task-completion contract changed. using var verify = new SqliteConnection($"Data Source={dataSource};Cache=Shared"); verify.Open(); using var countCmd = verify.CreateCommand(); countCmd.CommandText = "SELECT COUNT(*) FROM audit_event;"; Assert.Equal(0L, (long)countCmd.ExecuteScalar()!); } /// Verifies that the fallback writer rings every faulted row of a mid-batch insert failure and increments the failure counter once per row. /// A task that represents the asynchronous operation. [Fact] public async Task FallbackAuditWriter_MidBatchInsertFailure_RingsEveryFaultedRow_AndIncrementsFailureCounterForEach() { // End-to-end proof the fix actually closes the loop the finding describes: // FallbackAuditWriter composes the REAL SqliteAuditWriter as its primary. Before the // fix, evt1's primary write would appear to SUCCEED (its Task completed before the // rollback), so FallbackAuditWriter would never route it to the ring or count it as a // failure — the whole batch would be silently lost with SiteAuditWriteFailures staying // at 0, exactly as the finding describes. var (writer, _) = CreateWriter( nameof(FallbackAuditWriter_MidBatchInsertFailure_RingsEveryFaultedRow_AndIncrementsFailureCounterForEach)); await using var _w = writer; InstallFailureTrigger(writer); var ring = new RingBufferFallback(capacity: 16); var counter = Substitute.For(); var fallback = new FallbackAuditWriter( writer, ring, counter, NullLogger.Instance); var evt1 = NewEvent("normal-row"); var evt2 = NewEvent(FailMarker); // FallbackAuditWriter.WriteAsync never throws (audit failures are always swallowed at // that layer) — fired concurrently, same reasoning as above, so both land in one // FlushBatch transaction. var t1 = fallback.WriteAsync(evt1); var t2 = fallback.WriteAsync(evt2); await Task.WhenAll(t1, t2).WaitAsync(TimeSpan.FromSeconds(10)); // Both rows — including the one whose own INSERT succeeded before the batch rolled // back — must have been routed to the ring and counted as a primary failure. Assert.Equal(2, ring.Count); counter.Received(2).Increment(); } }