fix(gateway): bound the session replay ring by bytes as well as count (mxaccessgw#172)
ci / nightly-windev (push) Has been skipped
ci / windows-x86 (push) Successful in 1m26s
ci / java (push) Successful in 2m28s
ci / portable (push) Failing after 4m48s

The replay ring was bounded only by ReplayBufferCapacity (1024 events) and
ReplayRetentionSeconds. Event size is unbounded up to the worker frame size,
so a burst of large-array events could pin gigabytes per session while the
count looked small.

New MxGateway:Events:ReplayBufferMaxBytes (default 64 MiB, 0 disables) caps
the summed serialized size (MxEvent.CalculateSize, measured only when the cap
is on). Over the cap the oldest entries are evicted, exactly like the count
cap, so a resuming subscriber sees the existing ReplayGap path; an event bigger
than the whole cap is not retained. Every eviction (count, age or byte) now
clears its slot, so an evicted event is released immediately instead of
staying referenced until the slot is reused.

The 64 MiB default is a judgement call: it never binds for scalar traffic
(1024 x a few hundred bytes) and only trims large-array bursts.

Tests: SessionEventDistributorTests byte-cap eviction, single oversized event,
cap 0 = count only (the first two fail with the cap enforcement removed);
validator rejects a negative value. Docs: GatewayConfiguration.md, Sessions.md,
DesignDecisions.md; appsettings.json carries the key.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 12:38:22 -05:00
parent 7b21f304e9
commit 07badd139e
12 changed files with 186 additions and 17 deletions
@@ -121,6 +121,7 @@ public sealed class GatewayOptionsTests
[InlineData("MxGateway:Sessions:LeaseSweepIntervalSeconds", "0", "MxGateway:Sessions:LeaseSweepIntervalSeconds must be greater than zero.")]
[InlineData("MxGateway:Sessions:DetachGraceSeconds", "-1", "MxGateway:Sessions:DetachGraceSeconds must be zero or greater (0 disables detach-grace retention).")]
[InlineData("MxGateway:Sessions:WorkerReadyWaitTimeoutMs", "-1", "MxGateway:Sessions:WorkerReadyWaitTimeoutMs must be greater than or equal to zero.")]
[InlineData("MxGateway:Events:ReplayBufferMaxBytes", "-1", "MxGateway:Events:ReplayBufferMaxBytes must be greater than or equal to zero.")]
[InlineData("MxGateway:Events:QueueCapacity", "0", "MxGateway:Events:QueueCapacity must be greater than zero.")]
[InlineData("MxGateway:Protocol:MaxGrpcMessageBytes", "0", "MxGateway:Protocol:MaxGrpcMessageBytes must be between")]
[InlineData("MxGateway:Authentication:PepperSecretName", "", "MxGateway:Authentication:PepperSecretName is required")]
@@ -186,6 +186,97 @@ public sealed class SessionEventDistributorTests
Assert.Equal(new ulong[] { 3, 4, 5 }, replay.Select(e => e.WorkerSequence));
}
/// <summary>
/// mxaccessgw#172: the byte cap evicts oldest-first once the retained events' summed
/// serialized size exceeds it, even though the count cap is nowhere near reached, and a
/// resume from before the retained window reports a gap.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task ReplayBuffer_OverByteCap_EvictsOldestFirst_AndReportsGap()
{
int eventBytes = LargeEvent(1).CalculateSize();
Channel<MxEvent> source = Channel.CreateUnbounded<MxEvent>();
await using SessionEventDistributor distributor = CreateDistributor(
source.Reader,
replayBufferCapacity: 100,
replayRetentionSeconds: 0,
replayBufferMaxBytes: (eventBytes * 3) + (eventBytes / 2));
await distributor.StartAsync(CancellationToken.None);
using IEventSubscriberLease lease = distributor.Register();
for (ulong sequence = 1; sequence <= 5; sequence++)
{
source.Writer.TryWrite(LargeEvent(sequence));
}
for (ulong sequence = 1; sequence <= 5; sequence++)
{
Assert.Equal(sequence, (await ReadOneAsync(lease.Reader)).WorkerSequence);
}
Assert.True(distributor.TryGetReplayFrom(0, out IReadOnlyList<MxEvent> replay, out bool gap));
Assert.True(gap);
Assert.Equal(new ulong[] { 3, 4, 5 }, replay.Select(e => e.WorkerSequence));
}
/// <summary>
/// mxaccessgw#172: an event larger than the whole byte cap is not retained at all (the
/// cap is strict), and a resume from before it reports a gap with nothing to replay.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task ReplayBuffer_SingleEventOverByteCap_RetainsNothing_AndReportsGap()
{
Channel<MxEvent> source = Channel.CreateUnbounded<MxEvent>();
await using SessionEventDistributor distributor = CreateDistributor(
source.Reader,
replayBufferCapacity: 100,
replayRetentionSeconds: 0,
replayBufferMaxBytes: LargeEvent(1).CalculateSize() - 1);
await distributor.StartAsync(CancellationToken.None);
using IEventSubscriberLease lease = distributor.Register();
source.Writer.TryWrite(LargeEvent(1));
Assert.Equal(1UL, (await ReadOneAsync(lease.Reader)).WorkerSequence);
Assert.True(distributor.TryGetReplayFrom(0, out IReadOnlyList<MxEvent> replay, out bool gap));
Assert.True(gap);
Assert.Empty(replay);
}
/// <summary>
/// mxaccessgw#172: a byte cap of <c>0</c> disables the byte dimension, leaving only the
/// count cap — the pre-cap behaviour.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task ReplayBuffer_ByteCapZero_BoundsByCountOnly()
{
Channel<MxEvent> source = Channel.CreateUnbounded<MxEvent>();
await using SessionEventDistributor distributor = CreateDistributor(
source.Reader,
replayBufferCapacity: 100,
replayRetentionSeconds: 0,
replayBufferMaxBytes: 0);
await distributor.StartAsync(CancellationToken.None);
using IEventSubscriberLease lease = distributor.Register();
for (ulong sequence = 1; sequence <= 5; sequence++)
{
source.Writer.TryWrite(LargeEvent(sequence));
}
for (ulong sequence = 1; sequence <= 5; sequence++)
{
Assert.Equal(sequence, (await ReadOneAsync(lease.Reader)).WorkerSequence);
}
Assert.True(distributor.TryGetReplayFrom(0, out IReadOnlyList<MxEvent> replay, out bool gap));
Assert.False(gap);
Assert.Equal(new ulong[] { 1, 2, 3, 4, 5 }, replay.Select(e => e.WorkerSequence));
}
/// <summary>
/// Appending far more events than the capacity wraps the circular buffer's head
/// index past the array boundary multiple times; the retained window stays the
@@ -1220,7 +1311,8 @@ public sealed class SessionEventDistributorTests
ChannelReader<MxEvent> source,
int replayBufferCapacity,
double replayRetentionSeconds,
TimeProvider? timeProvider = null)
TimeProvider? timeProvider = null,
long replayBufferMaxBytes = 0)
=> new(
"session-test",
ct => source.ReadAllAsync(ct),
@@ -1228,11 +1320,17 @@ public sealed class SessionEventDistributorTests
replayBufferCapacity: replayBufferCapacity,
replayRetentionSeconds: replayRetentionSeconds,
NullLogger<SessionEventDistributor>.Instance,
timeProvider ?? TimeProvider.System);
timeProvider ?? TimeProvider.System,
replayBufferMaxBytes: replayBufferMaxBytes);
private static MxEvent Event(ulong sequence)
=> new() { SessionId = "session-test", WorkerSequence = sequence };
// A ~4 KB event whose serialized size is the same for every sequence below 128 (the varint
// for WorkerSequence stays one byte), so byte-cap arithmetic in the tests is exact.
private static MxEvent LargeEvent(ulong sequence)
=> new() { SessionId = new string('x', 4096), WorkerSequence = sequence };
private static async Task<MxEvent> ReadOneAsync(ChannelReader<MxEvent> reader)
{
await reader.WaitToReadAsync().AsTask().WaitAsync(ReadTimeout);