test(historian): pin the failed-batch-append branch; correct its loss-accounting docs
The batched ingest's append-failure catch had zero coverage, and two doc
statements overstated the loss: the Warning said the burst's values were
"dropped", and RecorderStatus.OutboxAppendFailures said "the value was
dropped, not recorded". The outbox contract is prefix-never-orphan, so a
partial-prefix failure leaves entries durably queued that drain normally.
- New test pins the branch: one 3-item burst whose group commit commits a
1-entry prefix then throws => OutboxAppendFailures == 1 (per burst, not
per value), TotalRecorded == 0, actor alive, and the prefix still reaches
the writer via the post-failure DrainTick (drainInterval 30s, so the
steady cadence cannot account for it). Falsified twice: counting per item
and dropping the drain nudge each turn it red.
- Second test pins the OperationCanceledException filter from its
observable side - a cancel that is NOT shutdown is metered, not swallowed.
Removing the 'when (_lifetime.IsCancellationRequested)' turns it red. The
shutdown side has no cheap seam (the counter dies with the actor).
- Docs corrected: OutboxAppendFailures counts FAILED BATCH APPENDS, one per
burst whatever its size (a granularity change from the per-value era), and
is not a count of values lost; TotalRecorded under-counts when a commit
fails after a partial prefix. Log message says "failed to append" and
names the prefix. No functional change beyond the message string.
Also folds in the DrainBatchSize default alignment behind 0298a0b9: the
recorder's two ctor defaults and its non-positive fallback still said 64
while ContinuousHistorizationOptions.DrainBatchSize now defaults to 1024.
No test pinned the old 64.
This commit is contained in:
+186
@@ -937,6 +937,140 @@ public sealed class ContinuousHistorizationRecorderTests : TestKit
|
||||
mux.ExpectNoMsg(TimeSpan.FromMilliseconds(300)); // no PreStart re-register == no restart
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A failed group commit is metered ONCE for the whole burst, credits nothing to
|
||||
/// <c>TotalRecorded</c>, leaves the actor alive, and STILL nudges the drain — which flushes whatever
|
||||
/// prefix the outbox durably committed before it failed.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Pins the batched ingest's append-failure catch branch, which had no coverage at all: every other
|
||||
/// test here uses an outbox that always succeeds, so a catch that miscounted, swallowed the drain
|
||||
/// nudge, or rethrew into supervision would go unnoticed.
|
||||
/// <para>
|
||||
/// Three separate claims, each of which is a distinct way the branch can be wrong:
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// (1) <c>OutboxAppendFailures == 1</c> for a THREE-item burst. The counter is per failed batch,
|
||||
/// not per value — a deliberate granularity change from the per-value era — so a catch that
|
||||
/// credited <c>entries.Count</c> would report 3 and make an operator read one bad fsync as three
|
||||
/// separate faults.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// (2) <c>TotalRecorded == 0</c> even though entry "A" IS durable. The outbox does not report how
|
||||
/// far it got, so the recorder deliberately under-counts rather than over-claims; this asserts
|
||||
/// the documented direction of the inaccuracy.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// (3) The durable prefix reaches the writer. This is the load-bearing half — it is the positive
|
||||
/// observation that the post-failure <c>DrainTick</c> was sent, and it is why the log message
|
||||
/// must not claim the burst was "dropped". <c>drainInterval</c> is set to 30 s so the steady
|
||||
/// cadence cannot fire inside the test: the only thing that can produce a drain here is the
|
||||
/// nudge. Deleting <c>self.Tell(DrainTick.Instance)</c> from the failure path (or returning from
|
||||
/// the catch) leaves "A" queued forever and turns this red.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
[Fact]
|
||||
public async Task A_failed_batch_append_counts_one_failure_and_still_ticks_the_drain()
|
||||
{
|
||||
var mux = CreateTestProbe();
|
||||
var writer = new FakeValueWriter { Succeed = true };
|
||||
var outbox = new InMemoryOutbox
|
||||
{
|
||||
// The partial-prefix shape: "A" commits durably, then the commit fails — so "B" and "C" are
|
||||
// genuinely lost while "A" is merely un-drained.
|
||||
AppendBatchFault = new IOException("simulated fsync failure (disk full)"),
|
||||
AppendBatchFaultAfter = 1,
|
||||
};
|
||||
|
||||
IActorRef rec = Sys.ActorOf(ContinuousHistorizationRecorder.Props(
|
||||
mux.Ref, writer, outbox, historizedRefs: new[] { "A", "B", "C" },
|
||||
drainInterval: TimeSpan.FromSeconds(30)));
|
||||
mux.ExpectMsg<DependencyMuxActor.RegisterInterest>();
|
||||
|
||||
var stamp = new DateTime(2026, 8, 15, 13, 0, 0, DateTimeKind.Utc);
|
||||
rec.Tell(new VirtualTagActor.DependencyValueBatch(new[]
|
||||
{
|
||||
new VirtualTagActor.DependencyValueChanged("A", 1.0, stamp),
|
||||
new VirtualTagActor.DependencyValueChanged("B", 2.0, stamp),
|
||||
new VirtualTagActor.DependencyValueChanged("C", 3.0, stamp),
|
||||
}));
|
||||
|
||||
// Answering GetStatus at all is itself the "actor is still alive" observation — an append failure
|
||||
// that reached supervision would have restarted (or stopped) it.
|
||||
await AwaitAssertAsync(
|
||||
async () =>
|
||||
{
|
||||
var status = await rec.Ask<ContinuousHistorizationRecorder.RecorderStatus>(
|
||||
ContinuousHistorizationRecorder.GetStatus.Instance, TimeSpan.FromSeconds(3));
|
||||
Assert.Equal(1, status.OutboxAppendFailures);
|
||||
Assert.Equal(0, status.TotalRecorded);
|
||||
},
|
||||
TimeSpan.FromSeconds(10));
|
||||
|
||||
await AwaitAssertAsync(
|
||||
() =>
|
||||
{
|
||||
Assert.True(outbox.PeekBatchCallCount >= 1, "the failure must still have nudged a drain");
|
||||
Assert.Contains(writer.Snapshot(), w => w.Tag == "A" && w.Value == 1.0);
|
||||
return Task.CompletedTask;
|
||||
},
|
||||
TimeSpan.FromSeconds(10));
|
||||
|
||||
// What never committed stays gone — the prefix drained, the remainder did not resurrect.
|
||||
Assert.DoesNotContain(writer.Snapshot(), w => w.Tag == "B");
|
||||
Assert.DoesNotContain(writer.Snapshot(), w => w.Tag == "C");
|
||||
mux.ExpectNoMsg(TimeSpan.FromMilliseconds(300)); // no PreStart re-register == no restart
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// An <see cref="OperationCanceledException"/> that is NOT shutdown is metered as an append failure,
|
||||
/// not swallowed.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The catch that quietly discards a cancelled append is filtered on
|
||||
/// <c>when (_lifetime.IsCancellationRequested)</c> — i.e. only during the actor's own shutdown.
|
||||
/// Drop the filter and every cancelled append becomes invisible: no metric, and no drain nudge for
|
||||
/// the prefix. This drives the observable side of that filter (the lifetime is open here, so the
|
||||
/// general catch must take it).
|
||||
/// <para>
|
||||
/// The shutdown side itself is deliberately not tested: <c>_lifetime</c> is cancelled in
|
||||
/// <c>PostStop</c>, so reaching that branch means racing an append against a stopping actor —
|
||||
/// and once it has stopped there is no <c>GetStatus</c> to read the counter from. There is no
|
||||
/// cheap seam for it; the filter's correctness is pinned from this side instead.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
[Fact]
|
||||
public async Task A_cancellation_that_is_not_shutdown_is_metered_as_an_append_failure()
|
||||
{
|
||||
var mux = CreateTestProbe();
|
||||
var writer = new FakeValueWriter { Succeed = true };
|
||||
var outbox = new InMemoryOutbox
|
||||
{
|
||||
AppendBatchFault = new OperationCanceledException("not a shutdown"),
|
||||
AppendBatchFaultAfter = 0,
|
||||
};
|
||||
|
||||
IActorRef rec = Sys.ActorOf(ContinuousHistorizationRecorder.Props(
|
||||
mux.Ref, writer, outbox, historizedRefs: new[] { "A" },
|
||||
drainInterval: TimeSpan.FromSeconds(30)));
|
||||
mux.ExpectMsg<DependencyMuxActor.RegisterInterest>();
|
||||
|
||||
rec.Tell(new VirtualTagActor.DependencyValueChanged(
|
||||
"A", 1.0, new DateTime(2026, 8, 15, 14, 0, 0, DateTimeKind.Utc)));
|
||||
|
||||
await AwaitAssertAsync(
|
||||
async () =>
|
||||
{
|
||||
var status = await rec.Ask<ContinuousHistorizationRecorder.RecorderStatus>(
|
||||
ContinuousHistorizationRecorder.GetStatus.Instance, TimeSpan.FromSeconds(3));
|
||||
Assert.Equal(1, status.OutboxAppendFailures);
|
||||
Assert.Equal(0, status.TotalRecorded);
|
||||
},
|
||||
TimeSpan.FromSeconds(10));
|
||||
|
||||
mux.ExpectNoMsg(TimeSpan.FromMilliseconds(300)); // no PreStart re-register == no restart
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="IHistorianValueWriter"/> double that BLOCKS every call until <see cref="Release"/>,
|
||||
/// recording the peak number of calls in flight simultaneously. That peak is what distinguishes a
|
||||
@@ -1097,6 +1231,34 @@ public sealed class ContinuousHistorizationRecorderTests : TestKit
|
||||
private int _appendBatchCalls;
|
||||
|
||||
private readonly bool _yieldOnAppend;
|
||||
private int _peekCalls;
|
||||
|
||||
/// <summary>When non-null, <see cref="AppendBatchAsync"/> durably appends the first
|
||||
/// <see cref="AppendBatchFaultAfter"/> entries and then throws this — the production outbox's
|
||||
/// "prefix, never orphan" contract on its failure path (an fsync hitting disk-full mid-commit).
|
||||
/// The throw happens AFTER a pool-thread yield, as a real group-commit fsync does, so the
|
||||
/// recorder's <c>ConfigureAwait(false)</c> continuation runs its catch block off the actor
|
||||
/// thread — the Gitea #491 shape, on the failure path.</summary>
|
||||
public Exception? AppendBatchFault { get; init; }
|
||||
|
||||
/// <summary>How many of the batch's entries are durably committed before
|
||||
/// <see cref="AppendBatchFault"/> is thrown. 0 = nothing committed; a positive value models the
|
||||
/// PARTIAL-PREFIX failure, where some entries are durable and will drain even though the append
|
||||
/// as a whole reported failure.</summary>
|
||||
public int AppendBatchFaultAfter { get; init; }
|
||||
|
||||
/// <summary>Lifetime count of <see cref="PeekBatchAsync"/> calls — one per drain pass, so a
|
||||
/// non-zero value is positive evidence that a drain actually RAN.</summary>
|
||||
public int PeekBatchCallCount
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _peekCalls;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <param name="capacity">Drop-oldest capacity; 0 means unbounded.</param>
|
||||
/// <param name="yieldOnAppend">
|
||||
@@ -1175,6 +1337,11 @@ public sealed class ContinuousHistorizationRecorderTests : TestKit
|
||||
_appendBatchCalls++;
|
||||
}
|
||||
|
||||
if (AppendBatchFault is not null)
|
||||
{
|
||||
return AppendBatchFaultingAsync(entries);
|
||||
}
|
||||
|
||||
if (entries.Count == 0)
|
||||
{
|
||||
return ValueTask.CompletedTask;
|
||||
@@ -1193,6 +1360,24 @@ public sealed class ContinuousHistorizationRecorderTests : TestKit
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
private async ValueTask AppendBatchFaultingAsync(IReadOnlyList<HistorizationOutboxEntry> entries)
|
||||
{
|
||||
// Complete on a POOL thread before faulting (the same requirement AppendYieldingAsync documents):
|
||||
// a real group commit fsyncs, so the recorder's ConfigureAwait(false) continuation — here, its
|
||||
// CATCH block — resumes with no ActorContext installed. A synchronous throw would run the catch
|
||||
// inline on the actor thread and would not exercise that.
|
||||
await Task.Delay(1).ConfigureAwait(false);
|
||||
|
||||
// "Prefix, never orphan": whatever was committed before the failure is durably tracked and
|
||||
// ackable, which is exactly what makes the post-failure drain nudge meaningful.
|
||||
for (var i = 0; i < AppendBatchFaultAfter && i < entries.Count; i++)
|
||||
{
|
||||
AppendCore(entries[i]);
|
||||
}
|
||||
|
||||
throw AppendBatchFault!;
|
||||
}
|
||||
|
||||
private async ValueTask AppendBatchYieldingAsync(IReadOnlyList<HistorizationOutboxEntry> entries)
|
||||
{
|
||||
// Same pool-thread-completion requirement as AppendYieldingAsync below — a group commit
|
||||
@@ -1235,6 +1420,7 @@ public sealed class ContinuousHistorizationRecorderTests : TestKit
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_peekCalls++;
|
||||
IReadOnlyList<HistorizationOutboxEntry> batch = _entries.Take(max).ToArray();
|
||||
return ValueTask.FromResult(batch);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user