Merge branch 'fix/gwc-25-replaygap-trio'

# Conflicts:
#	archreview/2026-07-12/remediation/00-tracking.md
#	archreview/2026-07-12/remediation/10-gateway-core.md

src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionEventDistributor.cs

@@ -390,10 +390,15 @@ public sealed class SessionEventDistributor : IAsyncDisposable
     ///     <see cref="TryGetReplayFrom"/> gap semantics.
     /// </param>
     /// <param name="oldestAvailableSequence">
-    ///     The oldest worker sequence still retained and replayable. <c>0</c> when nothing is
-    ///     retained. Meaningful to the caller only when <paramref name="gap"/> is
-    ///     <see langword="true"/> (it populates the ReplayGap sentinel's
-    ///     <c>oldest_available_sequence</c>).
+    ///     The resume anchor reported to a gapped client: the oldest worker sequence still
+    ///     retained and replayable, or — when age/capacity eviction has emptied the ring
+    ///     entirely — the next sequence that can possibly be delivered (highest observed + 1).
+    ///     Either way the client's documented
+    ///     <c>after_worker_sequence = oldest_available_sequence - 1</c> formula yields a cursor
+    ///     that resumes without dropping live events. <c>0</c> when <paramref name="gap"/> is
+    ///     <see langword="false"/>, where the value is meaningless and never emitted. Meaningful
+    ///     to the caller only when <paramref name="gap"/> is <see langword="true"/> (it populates
+    ///     the ReplayGap sentinel's <c>oldest_available_sequence</c>).
     /// </param>
     /// <param name="liveResumeSequence">
     ///     The worker sequence the live channel must resume strictly after: the highest
@@ -463,7 +468,20 @@ public sealed class SessionEventDistributor : IAsyncDisposable
             if (_replayCount == 0)
             {
                 gap = _anyEventSeen && afterSequence < _highestSequenceSeen;
-                oldestAvailableSequence = 0; // meaningful only when gap == true; 0 here since nothing is retained
+
+                // GWC-25: nothing is retained, but a gapped client still needs a usable resume
+                // anchor. The documented client formula is
+                // after_worker_sequence = oldest_available_sequence - 1, so reporting 0 here made
+                // an unsigned client compute ulong.MaxValue: the follow-up resume then replayed
+                // nothing and reported no gap (MaxValue is below no real sequence, in this branch
+                // and in the retained branch's wrap guard alike), and the caller's live filter
+                // (sequence > liveResumeSequence) dropped every subsequent event — a silently
+                // dead stream. Reporting the next sequence that can possibly
+                // be delivered (highest observed + 1) makes oldest - 1 land exactly on the
+                // highest observed sequence, so the resume delivers everything newer. Nothing is
+                // recoverable either way; the sentinel's job is to say "re-snapshot".
+                // Still 0 when gap == false, where the field is documented as meaningless.
+                oldestAvailableSequence = gap ? _highestSequenceSeen + 1 : 0;
             }
             else
             {

src/ZB.MOM.WW.MxGateway.Tests/Gateway/GatewayEndToEndReconnectReplayTests.cs

@@ -2,6 +2,7 @@ using Google.Protobuf.WellKnownTypes;
 using Grpc.Core;
 using Microsoft.Extensions.Logging.Abstractions;
 using Microsoft.Extensions.Options;
+using Microsoft.Extensions.Time.Testing;
 using ZB.MOM.WW.MxGateway.Contracts;
 using ZB.MOM.WW.MxGateway.Contracts.Proto;
 using ZB.MOM.WW.MxGateway.Server.Configuration;
@@ -231,6 +232,114 @@ public sealed class GatewayEndToEndReconnectReplayTests
         Assert.Equal(expectedTail, tail.Select(e => e.WorkerSequence).ToArray());
     }
 
+    /// <summary>
+    ///     GWC-25 regression, end to end: when age eviction has emptied the replay ring, the
+    ///     <c>ReplayGap</c> sentinel carries the next deliverable sequence (highest observed + 1),
+    ///     so a client applying the documented
+    ///     <c>after_worker_sequence = oldest_available_sequence - 1</c> formula on its follow-up
+    ///     resume receives subsequent live events. Under the pre-fix sentinel value of <c>0</c>
+    ///     the formula wrapped to <see cref="ulong.MaxValue"/> and the resumed stream never
+    ///     delivered another event.
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task ReconnectAfterFullAgeEvictionResumesWithSentinelFormula()
+    {
+        const int firstBatch = 4;
+        const double retentionSeconds = 30;
+
+        FakeTimeProvider clock = new(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero));
+        GatedEventFakeWorkerProcessLauncher launcher = new();
+
+        // Capacity is ample; age is the eviction axis under test.
+        await using ReconnectReplayGatewayServiceFixture fixture = new(
+            launcher,
+            replayBufferCapacity: 16,
+            replayRetentionSeconds: retentionSeconds,
+            timeProvider: clock);
+
+        string sessionId = await OpenSessionAsync(fixture, "reconnect-aged-out");
+
+        using CancellationTokenSource writer1Cts = new();
+        RecordingServerStreamWriter<MxEvent> writer1 = new();
+        Task stream1Task = Task.Run(async () =>
+            await fixture.Service.StreamEvents(
+                new StreamEventsRequest { SessionId = sessionId },
+                writer1,
+                new TestServerCallContext(cancellationToken: writer1Cts.Token)));
+
+        await fixture.WaitForSubscriberCountAsync(sessionId, n: 1, TestTimeout);
+        await WireUpAdviseAsync(fixture, sessionId);
+
+        for (int i = 0; i < firstBatch; i++)
+        {
+            launcher.AllowNextEvent();
+        }
+
+        IReadOnlyList<MxEvent> batch1 = await writer1.WaitForMessageCountAsync(firstBatch, TestTimeout);
+        ulong[] batch1Sequences = batch1.Select(e => e.WorkerSequence).ToArray();
+
+        // The client's cursor is mid-batch: it detached before consuming the tail, so it is
+        // genuinely behind the highest sequence the distributor observed.
+        ulong staleCursor = batch1Sequences[1];
+        ulong highestSeen = batch1Sequences[firstBatch - 1];
+
+        await DetachAsync(writer1Cts, stream1Task);
+
+        // Age every retained event out of the ring: the next resume finds it empty.
+        clock.Advance(TimeSpan.FromSeconds(retentionSeconds * 2));
+
+        // ---- first reconnect: receives only the sentinel, nothing is replayable ----
+        using CancellationTokenSource writer2Cts = new();
+        RecordingServerStreamWriter<MxEvent> writer2 = new();
+        Task stream2Task = Task.Run(async () =>
+            await fixture.Service.StreamEvents(
+                new StreamEventsRequest { SessionId = sessionId, AfterWorkerSequence = staleCursor },
+                writer2,
+                new TestServerCallContext(cancellationToken: writer2Cts.Token)));
+
+        await fixture.WaitForSubscriberCountAsync(sessionId, n: 1, TestTimeout);
+        IReadOnlyList<MxEvent> gapOnly = await writer2.WaitForMessageCountAsync(1, TestTimeout);
+
+        MxEvent sentinel = gapOnly[0];
+        Assert.NotNull(sentinel.ReplayGap);
+        Assert.Equal(staleCursor, sentinel.ReplayGap.RequestedAfterSequence);
+
+        // The whole point of GWC-25: the sentinel anchors the resume at the next sequence that
+        // can still be delivered, not at 0.
+        Assert.Equal(highestSeen + 1, sentinel.ReplayGap.OldestAvailableSequence);
+
+        await DetachAsync(writer2Cts, stream2Task);
+
+        // ---- second reconnect: the client applies the documented oldest - 1 formula ----
+        // Unchecked ulong arithmetic: with the pre-fix sentinel value of 0 this wraps to
+        // ulong.MaxValue and the live filter drops every subsequent event.
+        ulong resumeCursor = sentinel.ReplayGap.OldestAvailableSequence - 1;
+
+        RecordingServerStreamWriter<MxEvent> writer3 = new();
+        Task stream3Task = Task.Run(async () =>
+            await fixture.Service.StreamEvents(
+                new StreamEventsRequest { SessionId = sessionId, AfterWorkerSequence = resumeCursor },
+                writer3,
+                new TestServerCallContext()));
+
+        await fixture.WaitForSubscriberCountAsync(sessionId, n: 1, TestTimeout);
+
+        launcher.AllowNextEvent();
+        IReadOnlyList<MxEvent> live = await writer3.WaitForMessageCountAsync(1, TestTimeout);
+
+        launcher.StopEmitting();
+        await CloseAndDrainAsync(fixture, sessionId, stream3Task, launcher);
+
+        // A live event arrives on the resumed stream, and it is a real event, not another gap.
+        MxEvent delivered = live[0];
+        Assert.Null(delivered.ReplayGap);
+        Assert.Equal(MxEventFamily.OnDataChange, delivered.Family);
+        Assert.True(
+            delivered.WorkerSequence > highestSeen,
+            $"Live event {delivered.WorkerSequence} must be newer than the highest previously seen {highestSeen}.");
+    }
+
     // ---- shared flow helpers ----
 
     private static async Task<string> OpenSessionAsync(
@@ -385,11 +494,23 @@ public sealed class GatewayEndToEndReconnectReplayTests
         /// <summary>Initializes a new instance of the <see cref="ReconnectReplayGatewayServiceFixture"/> class.</summary>
         /// <param name="launcher">Fake worker process launcher backing the session manager.</param>
         /// <param name="replayBufferCapacity">Replay ring capacity for the session's event distributor.</param>
+        /// <param name="replayRetentionSeconds">
+        ///     Replay retention window. The default keeps age-eviction effectively off for the
+        ///     duration of a fast test; the age-eviction test shortens it and drives
+        ///     <paramref name="timeProvider"/>.
+        /// </param>
+        /// <param name="timeProvider">
+        ///     Clock handed to the session manager, which flows through to the session's event
+        ///     distributor. Pass a fake to make age-eviction deterministic.
+        /// </param>
         public ReconnectReplayGatewayServiceFixture(
             IWorkerProcessLauncher launcher,
-            int replayBufferCapacity)
+            int replayBufferCapacity,
+            double replayRetentionSeconds = 300,
+            TimeProvider? timeProvider = null)
         {
-            IOptions<GatewayOptions> options = Options.Create(CreateOptions(replayBufferCapacity));
+            IOptions<GatewayOptions> options = Options.Create(
+                CreateOptions(replayBufferCapacity, replayRetentionSeconds));
             SessionWorkerClientFactory workerClientFactory = new(
                 launcher,
                 options,
@@ -401,6 +522,7 @@ public sealed class GatewayEndToEndReconnectReplayTests
                 options,
                 _metrics,
                 logger: NullLogger<SessionManager>.Instance,
+                timeProvider: timeProvider,
                 dashboardEventBroadcaster: NullDashboardEventBroadcaster.Instance);
             MxAccessGrpcMapper mapper = new();
             EventStreamService eventStreamService = new(
@@ -472,7 +594,7 @@ public sealed class GatewayEndToEndReconnectReplayTests
             _metrics.Dispose();
         }
 
-        private static GatewayOptions CreateOptions(int replayBufferCapacity) =>
+        private static GatewayOptions CreateOptions(int replayBufferCapacity, double replayRetentionSeconds) =>
             new()
             {
                 Worker = new WorkerOptions
@@ -501,9 +623,10 @@ public sealed class GatewayEndToEndReconnectReplayTests
                     QueueCapacity = 32,
                     ReplayBufferCapacity = replayBufferCapacity,
 
-                    // Keep age-eviction effectively off for the duration of a fast test so
-                    // capacity is the only eviction axis under test.
-                    ReplayRetentionSeconds = 300,
+                    // Defaults to a window long enough that age-eviction never fires during a
+                    // fast test, so capacity is the only eviction axis; the age-eviction test
+                    // shortens it and drives a fake clock.
+                    ReplayRetentionSeconds = replayRetentionSeconds,
                 },
             };
     }

src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionEventDistributorTests.cs

@@ -841,6 +841,153 @@ public sealed class SessionEventDistributorTests
         Assert.Equal(4ul, live.WorkerSequence);
     }
 
+    /// <summary>
+    ///     GWC-25: when age eviction has emptied the replay ring, a resume behind the highest
+    ///     observed sequence still reports a gap, and the reported oldest-available sequence is
+    ///     the next sequence that can possibly be delivered (highest seen + 1) — never <c>0</c>,
+    ///     which would make the client's documented <c>oldest - 1</c> resume formula wrap to
+    ///     <see cref="ulong.MaxValue"/> and dead-stream the subscriber.
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task RegisterWithReplayReportsNextDeliverableSequenceWhenRingEmptiedByAge()
+    {
+        FakeTimeProvider time = new(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero));
+        Channel<MxEvent> source = Channel.CreateUnbounded<MxEvent>();
+        await using SessionEventDistributor distributor = CreateDistributor(
+            source.Reader,
+            replayBufferCapacity: 100,
+            replayRetentionSeconds: 30,
+            timeProvider: time);
+        await distributor.StartAsync(CancellationToken.None);
+
+        // A primer subscriber forces the pump to retain events 1..3 deterministically.
+        using IEventSubscriberLease primer = distributor.Register();
+        for (ulong sequence = 1; sequence <= 3; sequence++)
+        {
+            source.Writer.TryWrite(Event(sequence));
+            _ = await ReadOneAsync(primer.Reader);
+        }
+
+        // Past the retention window: RegisterWithReplay's EvictAged() empties the ring entirely.
+        time.Advance(TimeSpan.FromSeconds(60));
+
+        using IEventSubscriberLease resume = distributor.RegisterWithReplay(
+            1,
+            out IReadOnlyList<MxEvent> replay,
+            out bool gap,
+            out ulong oldestAvailable,
+            out ulong liveResume);
+
+        Assert.True(gap);
+        Assert.Empty(replay);
+        Assert.Equal(1ul, liveResume);
+
+        // Highest seen is 3, so the next deliverable sequence is 4.
+        Assert.Equal(4ul, oldestAvailable);
+    }
+
+    /// <summary>
+    ///     GWC-25: the same next-deliverable-sequence rule applies when retention is disabled
+    ///     outright (replay capacity 0) — the ring is always empty, so a behind cursor must still
+    ///     get a usable resume anchor rather than <c>0</c>.
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task RegisterWithReplayReportsNextDeliverableSequenceWithRetentionDisabled()
+    {
+        Channel<MxEvent> source = Channel.CreateUnbounded<MxEvent>();
+        await using SessionEventDistributor distributor = CreateDistributor(
+            source.Reader,
+            replayBufferCapacity: 0,
+            replayRetentionSeconds: 0);
+        await distributor.StartAsync(CancellationToken.None);
+
+        using IEventSubscriberLease primer = distributor.Register();
+        for (ulong sequence = 1; sequence <= 3; sequence++)
+        {
+            source.Writer.TryWrite(Event(sequence));
+            _ = await ReadOneAsync(primer.Reader);
+        }
+
+        using IEventSubscriberLease resume = distributor.RegisterWithReplay(
+            1,
+            out IReadOnlyList<MxEvent> replay,
+            out bool gap,
+            out ulong oldestAvailable,
+            out _);
+
+        Assert.True(gap);
+        Assert.Empty(replay);
+        Assert.Equal(4ul, oldestAvailable);
+    }
+
+    /// <summary>
+    ///     GWC-25 regression: a client that applies the documented
+    ///     <c>after_worker_sequence = oldest_available_sequence - 1</c> formula to an empty-ring
+    ///     ReplayGap resumes with a live watermark equal to the highest observed sequence, so the
+    ///     caller's live filter passes every subsequent event. Under the pre-fix <c>0</c> the
+    ///     formula wrapped to <see cref="ulong.MaxValue"/> and the stream went permanently silent.
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task ResumeUsingSentinelFormulaAfterEmptyRingGapDeliversLiveEvents()
+    {
+        FakeTimeProvider time = new(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero));
+        Channel<MxEvent> source = Channel.CreateUnbounded<MxEvent>();
+        await using SessionEventDistributor distributor = CreateDistributor(
+            source.Reader,
+            replayBufferCapacity: 100,
+            replayRetentionSeconds: 30,
+            timeProvider: time);
+        await distributor.StartAsync(CancellationToken.None);
+
+        using IEventSubscriberLease primer = distributor.Register();
+        for (ulong sequence = 1; sequence <= 3; sequence++)
+        {
+            source.Writer.TryWrite(Event(sequence));
+            _ = await ReadOneAsync(primer.Reader);
+        }
+
+        time.Advance(TimeSpan.FromSeconds(60));
+
+        // First resume: everything is evicted, so the sentinel carries the next deliverable
+        // sequence. The subscriber is disposed immediately — the client reconnects with the
+        // formula below.
+        IEventSubscriberLease gapped = distributor.RegisterWithReplay(
+            1,
+            out _,
+            out bool gap,
+            out ulong oldestAvailable,
+            out _);
+        gapped.Dispose();
+
+        Assert.True(gap);
+
+        // The documented client formula. Unchecked ulong arithmetic: under the pre-fix value of
+        // 0 this wraps to ulong.MaxValue, which is exactly the dead-stream defect.
+        ulong resumeCursor = oldestAvailable - 1;
+
+        using IEventSubscriberLease resumed = distributor.RegisterWithReplay(
+            resumeCursor,
+            out IReadOnlyList<MxEvent> replay,
+            out bool resumedGap,
+            out _,
+            out ulong liveResume);
+
+        Assert.False(resumedGap);
+        Assert.Empty(replay);
+
+        // The live filter the caller applies is "sequence > liveResume": it must sit at the
+        // highest observed sequence so the next event passes.
+        Assert.Equal(3ul, liveResume);
+
+        source.Writer.TryWrite(Event(4));
+        MxEvent live = await ReadOneAsync(resumed.Reader);
+        Assert.Equal(4ul, live.WorkerSequence);
+        Assert.True(live.WorkerSequence > liveResume);
+    }
+
     private static async Task DrainUntilFaultAsync(ChannelReader<MxEvent> reader)
     {
         // Drains any buffered events, then surfaces the channel's completion fault (if any)