perf(historian): group-commit batch append on the historization outbox
This commit is contained in:
+56
@@ -140,6 +140,62 @@ public sealed class FasterLogHistorizationOutboxTests : IDisposable
|
||||
Assert.Equal(new[] { a.Id, b.Id }, batch.Select(e => e.Id));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AppendBatch_then_peek_returns_fifo_in_batch_order()
|
||||
{
|
||||
// Group commit: one fsync for the whole batch. The reopen leg is the load-bearing half — it
|
||||
// proves the single per-batch commit actually made every entry durable, not just the last one.
|
||||
var dir = NewTempDir();
|
||||
var a = E("A", 1);
|
||||
var b = E("B", 2);
|
||||
var c = E("C", 3);
|
||||
{
|
||||
using var o = new FasterLogHistorizationOutbox(dir, HistorizationCommitMode.PerEntry);
|
||||
await o.AppendBatchAsync([a, b, c], TestContext.Current.CancellationToken);
|
||||
|
||||
Assert.Equal(3, await o.CountAsync(TestContext.Current.CancellationToken));
|
||||
var batch = await o.PeekBatchAsync(10, TestContext.Current.CancellationToken);
|
||||
Assert.Equal(new[] { a.Id, b.Id, c.Id }, batch.Select(e => e.Id));
|
||||
}
|
||||
|
||||
using var reopened = new FasterLogHistorizationOutbox(dir, HistorizationCommitMode.PerEntry);
|
||||
Assert.Equal(3, await reopened.CountAsync(TestContext.Current.CancellationToken));
|
||||
var survived = await reopened.PeekBatchAsync(10, TestContext.Current.CancellationToken);
|
||||
Assert.Equal(new[] { a.Id, b.Id, c.Id }, survived.Select(e => e.Id));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AppendBatch_at_capacity_drops_oldest_across_the_whole_batch()
|
||||
{
|
||||
// The drop-oldest loop runs ONCE for the batch, so an over-capacity batch must shed every
|
||||
// surplus entry (not just one) and count each drop.
|
||||
var dir = NewTempDir();
|
||||
using var o = new FasterLogHistorizationOutbox(dir, HistorizationCommitMode.PerEntry, capacity: 2);
|
||||
var a = E("A", 1);
|
||||
var b = E("B", 2);
|
||||
var c = E("C", 3);
|
||||
var d = E("D", 4);
|
||||
await o.AppendBatchAsync([a, b, c, d], TestContext.Current.CancellationToken);
|
||||
|
||||
Assert.Equal(2, await o.CountAsync(TestContext.Current.CancellationToken));
|
||||
Assert.Equal(2, o.DroppedCount);
|
||||
var tags = (await o.PeekBatchAsync(10, TestContext.Current.CancellationToken)).Select(e => e.Tag).ToArray();
|
||||
Assert.Equal(new[] { "C", "D" }, tags);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AppendBatch_empty_is_noop()
|
||||
{
|
||||
var dir = NewTempDir();
|
||||
using var o = new FasterLogHistorizationOutbox(dir, HistorizationCommitMode.PerEntry);
|
||||
await o.AppendAsync(E("A", 1), TestContext.Current.CancellationToken);
|
||||
|
||||
await o.AppendBatchAsync([], TestContext.Current.CancellationToken);
|
||||
|
||||
Assert.Equal(1, await o.CountAsync(TestContext.Current.CancellationToken));
|
||||
Assert.Equal(0, o.DroppedCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Remove_unknown_id_is_noop()
|
||||
{
|
||||
|
||||
+31
@@ -733,6 +733,37 @@ public sealed class ContinuousHistorizationRecorderTests : TestKit
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask AppendBatchAsync(IReadOnlyList<HistorizationOutboxEntry> entries, CancellationToken ct)
|
||||
{
|
||||
if (entries.Count == 0)
|
||||
{
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
if (_yieldOnAppend)
|
||||
{
|
||||
return AppendBatchYieldingAsync(entries);
|
||||
}
|
||||
|
||||
foreach (HistorizationOutboxEntry entry in entries)
|
||||
{
|
||||
AppendCore(entry);
|
||||
}
|
||||
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
private async ValueTask AppendBatchYieldingAsync(IReadOnlyList<HistorizationOutboxEntry> entries)
|
||||
{
|
||||
// Same pool-thread-completion requirement as AppendYieldingAsync below — a group commit
|
||||
// fsyncs once for the batch, so the yield happens once, before the entries are recorded.
|
||||
await Task.Delay(1).ConfigureAwait(false);
|
||||
foreach (HistorizationOutboxEntry entry in entries)
|
||||
{
|
||||
AppendCore(entry);
|
||||
}
|
||||
}
|
||||
|
||||
private async ValueTask AppendYieldingAsync(HistorizationOutboxEntry entry)
|
||||
{
|
||||
// Must complete on a POOL thread, carrying no synchronization context — that is what a real
|
||||
|
||||
@@ -245,6 +245,9 @@ public sealed class ServiceCollectionExtensionsTests
|
||||
public long DroppedCount => 0;
|
||||
/// <summary>No-op append.</summary>
|
||||
public ValueTask AppendAsync(HistorizationOutboxEntry entry, CancellationToken ct) => ValueTask.CompletedTask;
|
||||
/// <summary>No-op batch append.</summary>
|
||||
public ValueTask AppendBatchAsync(IReadOnlyList<HistorizationOutboxEntry> entries, CancellationToken ct)
|
||||
=> ValueTask.CompletedTask;
|
||||
/// <summary>Always returns an empty batch.</summary>
|
||||
public ValueTask<IReadOnlyList<HistorizationOutboxEntry>> PeekBatchAsync(int max, CancellationToken ct)
|
||||
=> ValueTask.FromResult<IReadOnlyList<HistorizationOutboxEntry>>(Array.Empty<HistorizationOutboxEntry>());
|
||||
|
||||
Reference in New Issue
Block a user