fix(historian): incremental batch-append bookkeeping; genuine group-commit durability pin

2026-08-16 04:18:38 -04:00
parent fbe8aaad9d
commit 7320586512
3 changed files with 98 additions and 14 deletions
@@ -99,8 +99,8 @@ public sealed class FasterLogHistorizationOutbox : IHistorizationOutbox
    {
        ArgumentNullException.ThrowIfNull(entry);

-        // A one-entry batch is byte-for-byte the single-append path (one enqueue, one PerEntry
-        // commit, one bookkeeping pass), so there is no second copy of the drop-oldest policy.
+        // A one-entry batch is byte-for-byte the single-append path (one enqueue, one indexing step,
+        // one PerEntry commit), so there is no second copy of the drop-oldest policy.
        await AppendBatchAsync([entry], ct).ConfigureAwait(false);
    }

@@ -116,11 +116,24 @@ public sealed class FasterLogHistorizationOutbox : IHistorizationOutbox
        for (int i = 0; i < entries.Count; i++)
            ArgumentNullException.ThrowIfNull(entries[i]);

-        long[] startAddresses = new long[entries.Count];
+        // INCREMENTAL bookkeeping: each entry is indexed under _state immediately after ITS OWN
+        // enqueue returns — never accumulated and applied after the loop. That is what makes a
+        // mid-loop failure (cancellation, I/O fault) safe: entries 0..i-1 are then both physically in
+        // the log AND in _live/_index, so PeekBatchAsync (which scans the raw log) and RemoveAsync
+        // (which looks the id up in _index) agree about them, and the head can advance past them.
+        // Deferring the indexing until after the loop stranded that prefix physically-present but
+        // untracked — peekable, un-removable (RemoveAsync no-ops on an unknown id), hence redelivered
+        // to the historian on every drain pass and blocking truncation until a restart's
+        // RecoverState() re-indexed it — and it also escaped the capacity bound.
        for (int i = 0; i < entries.Count; i++)
        {
-            byte[] payload = HistorizationOutboxEntrySerializer.Serialize(entries[i]);
-            startAddresses[i] = await _log.EnqueueAsync(payload, ct).ConfigureAwait(false);
+            HistorizationOutboxEntry entry = entries[i];
+            byte[] payload = HistorizationOutboxEntrySerializer.Serialize(entry);
+            long start = await _log.EnqueueAsync(payload, ct).ConfigureAwait(false);
+            lock (_state)
+            {
+                _index[entry.Id] = _live.AddLast(new LiveEntry(entry.Id, start));
+            }
        }

        if (_commitMode == HistorizationCommitMode.PerEntry)
@@ -128,23 +141,26 @@ public sealed class FasterLogHistorizationOutbox : IHistorizationOutbox
            // Group commit: ONE fsync makes the whole batch durable before returning, replacing the
            // N per-entry fsyncs a loop of AppendAsync would have cost. Periodic skips this — the
            // background timer (and Dispose) commit on their cadence (accepted trade-off).
+            //
+            // Residual (documented on IHistorizationOutbox.AppendBatchAsync): a throw before this
+            // commit leaves the enqueued prefix tracked but not yet committed, exactly the exposure
+            // the single-entry path has always had between its enqueue and its commit. Such entries
+            // are served from memory until a restart, where RecoverState() rebuilds from the
+            // committed log and simply does not see them.
            await _log.CommitAsync(ct).ConfigureAwait(false);
        }

        long? truncateTo = null;
        lock (_state)
        {
-            for (int i = 0; i < entries.Count; i++)
-            {
-                HistorizationOutboxEntry entry = entries[i];
-                LinkedListNode<LiveEntry> node = _live.AddLast(new LiveEntry(entry.Id, startAddresses[i]));
-                _index[entry.Id] = node;
-            }
-
            // Drop-oldest on overflow, run ONCE for the whole batch (an over-capacity batch sheds
            // every surplus entry here). The new head is the start address of whatever entry
            // survives at the front (or the tail if the log emptied); truncate to the furthest such
            // address, so the batch needs at most one truncate + one extra commit.
+            //
+            // A batch that throws mid-loop skips this pass, so _live can sit transiently above
+            // _capacity — the same convergent overflow RecoverState() documents, shed by the next
+            // append's while-loop.
            while (_capacity > 0 && _live.Count > _capacity)
            {
                LinkedListNode<LiveEntry> oldest = _live.First!;