perf(historian): group-commit batch append on the historization outbox
This commit is contained in:
+37
-8
@@ -99,23 +99,52 @@ public sealed class FasterLogHistorizationOutbox : IHistorizationOutbox
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(entry);
|
||||
|
||||
byte[] payload = HistorizationOutboxEntrySerializer.Serialize(entry);
|
||||
long startAddress = await _log.EnqueueAsync(payload, ct).ConfigureAwait(false);
|
||||
// A one-entry batch is byte-for-byte the single-append path (one enqueue, one PerEntry
|
||||
// commit, one bookkeeping pass), so there is no second copy of the drop-oldest policy.
|
||||
await AppendBatchAsync([entry], ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async ValueTask AppendBatchAsync(IReadOnlyList<HistorizationOutboxEntry> entries, CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(entries);
|
||||
if (entries.Count == 0)
|
||||
return;
|
||||
|
||||
// Validate before ANY enqueue: a null halfway through would otherwise leave the batch's
|
||||
// earlier entries in the log with no in-memory index entry (invisible but not reclaimable).
|
||||
for (int i = 0; i < entries.Count; i++)
|
||||
ArgumentNullException.ThrowIfNull(entries[i]);
|
||||
|
||||
long[] startAddresses = new long[entries.Count];
|
||||
for (int i = 0; i < entries.Count; i++)
|
||||
{
|
||||
byte[] payload = HistorizationOutboxEntrySerializer.Serialize(entries[i]);
|
||||
startAddresses[i] = await _log.EnqueueAsync(payload, ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (_commitMode == HistorizationCommitMode.PerEntry)
|
||||
{
|
||||
// PerEntry: durable before returning. Periodic skips this — the background timer (and
|
||||
// Dispose) commit on their cadence (accepted throughput/latency trade-off).
|
||||
// Group commit: ONE fsync makes the whole batch durable before returning, replacing the
|
||||
// N per-entry fsyncs a loop of AppendAsync would have cost. Periodic skips this — the
|
||||
// background timer (and Dispose) commit on their cadence (accepted trade-off).
|
||||
await _log.CommitAsync(ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
long? truncateTo = null;
|
||||
lock (_state)
|
||||
{
|
||||
LinkedListNode<LiveEntry> node = _live.AddLast(new LiveEntry(entry.Id, startAddress));
|
||||
_index[entry.Id] = node;
|
||||
for (int i = 0; i < entries.Count; i++)
|
||||
{
|
||||
HistorizationOutboxEntry entry = entries[i];
|
||||
LinkedListNode<LiveEntry> node = _live.AddLast(new LiveEntry(entry.Id, startAddresses[i]));
|
||||
_index[entry.Id] = node;
|
||||
}
|
||||
|
||||
// Drop-oldest on overflow. The new head is the start address of whatever entry survives
|
||||
// at the front (or the tail if the log emptied); truncate to the furthest such address.
|
||||
// Drop-oldest on overflow, run ONCE for the whole batch (an over-capacity batch sheds
|
||||
// every surplus entry here). The new head is the start address of whatever entry
|
||||
// survives at the front (or the tail if the log emptied); truncate to the furthest such
|
||||
// address, so the batch needs at most one truncate + one extra commit.
|
||||
while (_capacity > 0 && _live.Count > _capacity)
|
||||
{
|
||||
LinkedListNode<LiveEntry> oldest = _live.First!;
|
||||
|
||||
Reference in New Issue
Block a user