feat(sessions): replay-on-reconnect with ReplayGap sentinel
This commit is contained in:
Joseph Doherty
@@ -287,30 +287,14 @@ public sealed class SessionEventDistributor : IAsyncDisposable
|
||||
/// </param>
|
||||
public IEventSubscriberLease Register(bool isInternal = false)
|
||||
{
|
||||
// The pump is the single writer for this channel; readers are single-consumer
|
||||
// (one gRPC stream / dashboard subscriber). Synchronous continuations are
|
||||
// disabled so a slow reader can never stall the pump on its completion.
|
||||
//
|
||||
// The pump MUST stay non-blocking: it writes with the non-blocking TryWrite so one
|
||||
// slow reader can never stall the single pump that feeds every subscriber. FullMode
|
||||
// is deliberately Wait — NOT because the pump ever blocks (it never calls the blocking
|
||||
// WriteAsync overload), but because Wait is the only BoundedChannelFullMode under
|
||||
// which TryWrite returns false when the channel is full. That false return IS the
|
||||
// overflow signal the pump needs to apply the per-subscriber backpressure policy. The
|
||||
// Drop* modes would make TryWrite silently succeed-and-drop, hiding overflow and
|
||||
// re-introducing the silent data loss this task removes. So: Wait mode + TryWrite =
|
||||
// a non-blocking pump that still detects a full subscriber channel.
|
||||
Channel<MxEvent> channel = Channel.CreateBounded<MxEvent>(
|
||||
new BoundedChannelOptions(_subscriberQueueCapacity)
|
||||
{
|
||||
SingleReader = true,
|
||||
SingleWriter = true,
|
||||
FullMode = BoundedChannelFullMode.Wait,
|
||||
AllowSynchronousContinuations = false,
|
||||
});
|
||||
|
||||
Channel<MxEvent> channel = CreateSubscriberChannel();
|
||||
long id = Interlocked.Increment(ref _nextSubscriberId);
|
||||
Subscriber subscriber = new(id, channel, isInternal);
|
||||
return RegisterSubscriber(subscriber);
|
||||
}
|
||||
|
||||
private IEventSubscriberLease RegisterSubscriber(Subscriber subscriber)
|
||||
{
|
||||
|
||||
// The disposed check AND the map add happen under the same lock with no await
|
||||
// in between. DisposeAsync sets _disposed=true under this same lock before it
|
||||
@@ -320,7 +304,152 @@ public sealed class SessionEventDistributor : IAsyncDisposable
|
||||
lock (_lifecycleLock)
|
||||
{
|
||||
ObjectDisposedException.ThrowIf(_disposed, this);
|
||||
_subscribers[id] = subscriber;
|
||||
_subscribers[subscriber.Id] = subscriber;
|
||||
}
|
||||
|
||||
return new SubscriberLease(this, subscriber);
|
||||
}
|
||||
|
||||
// Creates a per-subscriber bounded channel. The pump is the single writer; readers are
|
||||
// single-consumer (one gRPC stream / dashboard subscriber). Synchronous continuations are
|
||||
// disabled so a slow reader can never stall the pump on its completion.
|
||||
//
|
||||
// The pump MUST stay non-blocking: it writes with the non-blocking TryWrite so one slow
|
||||
// reader can never stall the single pump that feeds every subscriber. FullMode is
|
||||
// deliberately Wait — NOT because the pump ever blocks (it never calls the blocking
|
||||
// WriteAsync overload), but because Wait is the only BoundedChannelFullMode under which
|
||||
// TryWrite returns false when the channel is full. That false return IS the overflow signal
|
||||
// the pump needs to apply the per-subscriber backpressure policy. The Drop* modes would
|
||||
// make TryWrite silently succeed-and-drop, hiding overflow and re-introducing silent data
|
||||
// loss. So: Wait mode + TryWrite = a non-blocking pump that still detects a full channel.
|
||||
private Channel<MxEvent> CreateSubscriberChannel()
|
||||
=> Channel.CreateBounded<MxEvent>(
|
||||
new BoundedChannelOptions(_subscriberQueueCapacity)
|
||||
{
|
||||
SingleReader = true,
|
||||
SingleWriter = true,
|
||||
FullMode = BoundedChannelFullMode.Wait,
|
||||
AllowSynchronousContinuations = false,
|
||||
});
|
||||
|
||||
/// <summary>
|
||||
/// Atomically snapshots the replay ring for events newer than
|
||||
/// <paramref name="afterSequence"/> AND registers a live subscriber, so the
|
||||
/// replay→live handoff has no gap and no duplicate (Task 12 reconnect/resume).
|
||||
/// </summary>
|
||||
/// <param name="afterSequence">
|
||||
/// The last worker sequence the reconnecting client already observed. Replay returns
|
||||
/// events strictly newer than this; the live channel is filtered (by the caller) to
|
||||
/// events strictly newer than the last replayed sequence.
|
||||
/// </param>
|
||||
/// <param name="replayedEvents">
|
||||
/// The retained events newer than <paramref name="afterSequence"/>, in ascending
|
||||
/// sequence order. Never null; empty when nothing newer is retained.
|
||||
/// </param>
|
||||
/// <param name="gap">
|
||||
/// <see langword="true"/> when events between <paramref name="afterSequence"/> and the
|
||||
/// oldest retained event were already evicted (capacity/age), so the client missed
|
||||
/// events that can no longer be replayed and must re-snapshot. Mirrors
|
||||
/// <see cref="TryGetReplayFrom"/> gap semantics.
|
||||
/// </param>
|
||||
/// <param name="oldestAvailableSequence">
|
||||
/// The oldest worker sequence still retained and replayable. <c>0</c> when nothing is
|
||||
/// retained. Meaningful to the caller only when <paramref name="gap"/> is
|
||||
/// <see langword="true"/> (it populates the ReplayGap sentinel's
|
||||
/// <c>oldest_available_sequence</c>).
|
||||
/// </param>
|
||||
/// <param name="liveResumeSequence">
|
||||
/// The worker sequence the live channel must resume strictly after: the highest
|
||||
/// replayed sequence, or <paramref name="afterSequence"/> when nothing was replayed.
|
||||
/// The caller MUST apply this as the per-subscriber live filter so any event that was
|
||||
/// both replayed here and subsequently fanned into this subscriber's live channel is
|
||||
/// dropped exactly once (no duplicate), while every newer event is delivered (no gap).
|
||||
/// </param>
|
||||
/// <param name="isInternal">
|
||||
/// <see langword="true"/> for a gateway-owned internal subscriber. See
|
||||
/// <see cref="Register"/>.
|
||||
/// </param>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Why this is atomic and the handoff is correct.</b> The replay snapshot and the
|
||||
/// subscriber registration both run inside the SAME <c>_replayLock</c> critical
|
||||
/// section. The pump appends each event to the replay buffer under <c>_replayLock</c>
|
||||
/// <em>before</em> fanning it to subscribers (outside the lock). Therefore, relative
|
||||
/// to this method's critical section, for every event E:
|
||||
/// </para>
|
||||
/// <list type="bullet">
|
||||
/// <item>
|
||||
/// If the pump appended E before this critical section, E is in
|
||||
/// <paramref name="replayedEvents"/> (when newer than
|
||||
/// <paramref name="afterSequence"/>). The pump's fan-out of E may race the
|
||||
/// registration: if it writes E to this new channel too, E's sequence is
|
||||
/// <c><= liveResumeSequence</c>, so the caller's live filter DROPS it — no
|
||||
/// duplicate.
|
||||
/// </item>
|
||||
/// <item>
|
||||
/// If the pump appends E after this critical section, E is NOT in the snapshot,
|
||||
/// but this subscriber is already registered, so the pump fans E into the live
|
||||
/// channel with sequence <c>> liveResumeSequence</c> — delivered as live, no
|
||||
/// gap.
|
||||
/// </item>
|
||||
/// </list>
|
||||
/// <para>
|
||||
/// Lock ordering: this is the only path that holds both <c>_replayLock</c> and
|
||||
/// <c>_lifecycleLock</c>; it always takes <c>_replayLock</c> first then
|
||||
/// <c>_lifecycleLock</c>. No other path acquires both, so there is no inversion.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public IEventSubscriberLease RegisterWithReplay(
|
||||
ulong afterSequence,
|
||||
out IReadOnlyList<MxEvent> replayedEvents,
|
||||
out bool gap,
|
||||
out ulong oldestAvailableSequence,
|
||||
out ulong liveResumeSequence,
|
||||
bool isInternal = false)
|
||||
{
|
||||
Channel<MxEvent> channel = CreateSubscriberChannel();
|
||||
long id = Interlocked.Increment(ref _nextSubscriberId);
|
||||
Subscriber subscriber = new(id, channel, isInternal);
|
||||
|
||||
// Snapshot replay AND register under a single _replayLock section so the live channel
|
||||
// begins exactly where the replay snapshot ends — see the remarks for the no-gap /
|
||||
// no-duplicate argument. _lifecycleLock is nested inside (consistent ordering) only to
|
||||
// honor the disposed check and the same add semantics as Register.
|
||||
lock (_replayLock)
|
||||
{
|
||||
EvictAged();
|
||||
|
||||
List<MxEvent> newer = [];
|
||||
ulong highestReplayed = afterSequence;
|
||||
|
||||
if (_replayBuffer.Count == 0)
|
||||
{
|
||||
oldestAvailableSequence = 0;
|
||||
gap = _anyEventSeen && afterSequence < _highestSequenceSeen;
|
||||
}
|
||||
else
|
||||
{
|
||||
oldestAvailableSequence = _replayBuffer.First!.Value.Event.WorkerSequence;
|
||||
gap = oldestAvailableSequence > 0 && afterSequence < oldestAvailableSequence - 1;
|
||||
|
||||
foreach (ReplayEntry entry in _replayBuffer)
|
||||
{
|
||||
if (entry.Event.WorkerSequence > afterSequence)
|
||||
{
|
||||
newer.Add(entry.Event);
|
||||
highestReplayed = entry.Event.WorkerSequence;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
replayedEvents = newer;
|
||||
liveResumeSequence = highestReplayed;
|
||||
|
||||
lock (_lifecycleLock)
|
||||
{
|
||||
ObjectDisposedException.ThrowIf(_disposed, this);
|
||||
_subscribers[id] = subscriber;
|
||||
}
|
||||
}
|
||||
|
||||
return new SubscriberLease(this, subscriber);
|
||||
|
||||
Reference in New Issue
Block a user