perf(historian): group-commit batch append on the historization outbox

This commit is contained in:
Joseph Doherty
2026-08-15 21:40:04 -04:00
parent debd44c213
commit 6ff64ca067
5 changed files with 138 additions and 8 deletions
@@ -140,6 +140,62 @@ public sealed class FasterLogHistorizationOutboxTests : IDisposable
Assert.Equal(new[] { a.Id, b.Id }, batch.Select(e => e.Id));
}
[Fact]
public async Task AppendBatch_then_peek_returns_fifo_in_batch_order()
{
// Group commit: one fsync for the whole batch. The reopen leg is the load-bearing half — it
// proves the single per-batch commit actually made every entry durable, not just the last one.
var dir = NewTempDir();
var a = E("A", 1);
var b = E("B", 2);
var c = E("C", 3);
{
using var o = new FasterLogHistorizationOutbox(dir, HistorizationCommitMode.PerEntry);
await o.AppendBatchAsync([a, b, c], TestContext.Current.CancellationToken);
Assert.Equal(3, await o.CountAsync(TestContext.Current.CancellationToken));
var batch = await o.PeekBatchAsync(10, TestContext.Current.CancellationToken);
Assert.Equal(new[] { a.Id, b.Id, c.Id }, batch.Select(e => e.Id));
}
using var reopened = new FasterLogHistorizationOutbox(dir, HistorizationCommitMode.PerEntry);
Assert.Equal(3, await reopened.CountAsync(TestContext.Current.CancellationToken));
var survived = await reopened.PeekBatchAsync(10, TestContext.Current.CancellationToken);
Assert.Equal(new[] { a.Id, b.Id, c.Id }, survived.Select(e => e.Id));
}
[Fact]
public async Task AppendBatch_at_capacity_drops_oldest_across_the_whole_batch()
{
// The drop-oldest loop runs ONCE for the batch, so an over-capacity batch must shed every
// surplus entry (not just one) and count each drop.
var dir = NewTempDir();
using var o = new FasterLogHistorizationOutbox(dir, HistorizationCommitMode.PerEntry, capacity: 2);
var a = E("A", 1);
var b = E("B", 2);
var c = E("C", 3);
var d = E("D", 4);
await o.AppendBatchAsync([a, b, c, d], TestContext.Current.CancellationToken);
Assert.Equal(2, await o.CountAsync(TestContext.Current.CancellationToken));
Assert.Equal(2, o.DroppedCount);
var tags = (await o.PeekBatchAsync(10, TestContext.Current.CancellationToken)).Select(e => e.Tag).ToArray();
Assert.Equal(new[] { "C", "D" }, tags);
}
[Fact]
public async Task AppendBatch_empty_is_noop()
{
var dir = NewTempDir();
using var o = new FasterLogHistorizationOutbox(dir, HistorizationCommitMode.PerEntry);
await o.AppendAsync(E("A", 1), TestContext.Current.CancellationToken);
await o.AppendBatchAsync([], TestContext.Current.CancellationToken);
Assert.Equal(1, await o.CountAsync(TestContext.Current.CancellationToken));
Assert.Equal(0, o.DroppedCount);
}
[Fact]
public async Task Remove_unknown_id_is_noop()
{