chore(historian): batch-era defaults + appsettings/doc alignment
This commit is contained in:
@@ -43,10 +43,17 @@
|
||||
"OutboxPath": "",
|
||||
"_OutboxPathComment": "Directory holding the FasterLog segment + commit files. Required when Enabled=true. In production set an ABSOLUTE path on durable storage.",
|
||||
"CommitMode": "PerEntry",
|
||||
"_CommitModeComment": "PerEntry means one fsync per ingest BURST, not per value: the recorder appends a whole burst through AppendBatchAsync, which enqueues every entry and issues a single group commit. Periodic batches commits onto a timer every CommitIntervalMs (bounded loss window).",
|
||||
"CommitIntervalMs": 100,
|
||||
"DrainBatchSize": 64,
|
||||
"DrainBatchSize": 1024,
|
||||
"_DrainBatchSizeComment": "Entries peeked + written per drain pass. Was 64 when a pass wrote its per-tag groups sequentially; the peek is an in-memory scan and the pass is now bounded-parallel over DrainConcurrency, so a wide batch is cheap — and 64 let the outbox fall permanently behind during any gateway hiccup.",
|
||||
"DrainIntervalSeconds": 2,
|
||||
"Capacity": 0,
|
||||
"DrainConcurrency": 8,
|
||||
"_DrainConcurrencyComment": "Historian tags a single drain pass writes concurrently (one gateway round-trip per tag group). 1 restores the fully sequential drain. The ack is unchanged: still all-or-nothing after every group reports.",
|
||||
"Capacity": 500000,
|
||||
"_CapacityComment": "Max un-acked outbox entries before drop-oldest; 0 = unbounded. The Gitea #538 disk backstop for a historian refusing EVERY write — the one case the per-tag quarantine cannot resolve.",
|
||||
"MaxWriteAttempts": 5,
|
||||
"_MaxWriteAttemptsComment": "Gitea #538: consecutive failed drain passes one tag may accumulate — counted only across passes where OTHER tags wrote successfully — before its queued entries are abandoned so the FIFO head can advance.",
|
||||
"MinBackoffSeconds": 1,
|
||||
"MaxBackoffSeconds": 30
|
||||
},
|
||||
|
||||
@@ -43,6 +43,15 @@ public sealed class ContinuousHistorizationOptions
|
||||
/// every <see cref="CommitIntervalMs"/> ms (higher throughput, a bounded worst-case loss window).
|
||||
/// Parsed case-insensitively against <c>HistorizationCommitMode</c>; an unrecognized value falls
|
||||
/// back to <c>PerEntry</c>.
|
||||
/// <para>
|
||||
/// <b><c>PerEntry</c> now means one fsync per ingest <em>burst</em>, not per value.</b> The
|
||||
/// recorder appends a whole value burst through
|
||||
/// <c>IHistorizationOutbox.AppendBatchAsync</c>, which enqueues every entry and then issues a
|
||||
/// <em>single</em> group commit — so a burst of N values costs one fsync instead of N. The
|
||||
/// durability contract is unchanged: <c>AppendBatchAsync</c> does not return until that
|
||||
/// commit completes, so nothing the caller has been told is buffered can be lost to an
|
||||
/// unclean restart.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public string CommitMode { get; init; } = "PerEntry";
|
||||
|
||||
@@ -50,9 +59,23 @@ public sealed class ContinuousHistorizationOptions
|
||||
/// <see cref="CommitMode"/> is <c>Periodic</c>. Ignored under <c>PerEntry</c>.</summary>
|
||||
public int CommitIntervalMs { get; init; } = 100;
|
||||
|
||||
/// <summary>Maximum outbox entries peeked + written per drain pass; clamped to a positive value by
|
||||
/// the recorder (a non-positive value falls back to 64).</summary>
|
||||
public int DrainBatchSize { get; init; } = 64;
|
||||
/// <summary>
|
||||
/// Maximum outbox entries peeked + written per drain pass; clamped to a positive value by the
|
||||
/// recorder (a non-positive value falls back to the default).
|
||||
/// <para>
|
||||
/// Defaults to 1024. It was 64 when a pass wrote its per-tag groups <em>sequentially</em>, so
|
||||
/// a wide batch cost <c>tags × latency</c> on the drain thread. Both halves of that reasoning
|
||||
/// are gone: the peek is an in-memory scan of the outbox's live index (widening it costs
|
||||
/// allocation, not I/O), and the pass is now bounded-parallel over
|
||||
/// <see cref="DrainConcurrency"/>, so its cost is <c>⌈tags / concurrency⌉ × latency</c>
|
||||
/// regardless of batch width. At 64, a drain draws down at most 64 entries every
|
||||
/// <see cref="DrainIntervalSeconds"/> — below the arrival rate of even a modest historized
|
||||
/// tag set — so any gateway hiccup left the outbox permanently behind, growing toward
|
||||
/// <see cref="Capacity"/> and shedding oldest samples, rather than catching up once the
|
||||
/// gateway recovered.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public int DrainBatchSize { get; init; } = 1024;
|
||||
|
||||
/// <summary>Steady drain cadence in seconds (also the post-success reschedule). Defaults to 2.</summary>
|
||||
public double DrainIntervalSeconds { get; init; } = 2;
|
||||
|
||||
Reference in New Issue
Block a user