feat(sessions): multi-subscriber cap enforcement + mode-gated FailFast

2026-06-15 15:32:08 -04:00
parent 7b12eebbd1
commit ac42783e36
12 changed files with 399 additions and 81 deletions
@@ -151,7 +151,7 @@ public sealed class GatewaySessionDashboardMirrorTests
        session.MarkReady();

        Assert.Equal(0, session.ActiveEventSubscriberCount);
-        using IEventSubscriberLease lease = session.AttachEventSubscriber(allowMultipleSubscribers: false);
+        using IEventSubscriberLease lease = session.AttachEventSubscriber(allowMultipleSubscribers: false, maxSubscribers: 1);
        Assert.Equal(1, session.ActiveEventSubscriberCount);
    }

@@ -2,10 +2,12 @@ using System.Runtime.CompilerServices;
 using Microsoft.Extensions.Logging.Abstractions;
 using ZB.MOM.WW.MxGateway.Contracts.Proto;
 using ZB.MOM.WW.MxGateway.Server.Configuration;
+using ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
 using ZB.MOM.WW.MxGateway.Server.Grpc;
 using ZB.MOM.WW.MxGateway.Server.Metrics;
 using ZB.MOM.WW.MxGateway.Server.Sessions;
 using ZB.MOM.WW.MxGateway.Server.Workers;
+using ZB.MOM.WW.MxGateway.Tests.TestSupport;

 namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Sessions;

@@ -185,7 +187,7 @@ public sealed class GatewaySessionTests
        {
            // Attach one subscriber; this increments _activeEventSubscriberCount to 1.
            IEventSubscriberLease lease = session.AttachEventSubscriber(
-                allowMultipleSubscribers: false);
+                allowMultipleSubscribers: false, maxSubscribers: 1);

            // Race Concurrency threads all calling Dispose() on the same lease.
            // Only one must actually run DetachEventSubscriber.
@@ -211,7 +213,7 @@ public sealed class GatewaySessionTests
            // Observable contract: a fresh single subscriber must now be attachable
            // (i.e., the guard _activeEventSubscriberCount > 0 is false).
            IEventSubscriberLease next = session.AttachEventSubscriber(
-                allowMultipleSubscribers: false);
+                allowMultipleSubscribers: false, maxSubscribers: 1);
            next.Dispose();
            Assert.Equal(0, session.ActiveEventSubscriberCount);
        }
@@ -220,6 +222,220 @@ public sealed class GatewaySessionTests
        await session.DisposeAsync();
    }

+    /// <summary>
+    ///     Task 8 regression. Single-subscriber mode rejects a SECOND concurrent external
+    ///     attach with <see cref="SessionManagerErrorCode.EventSubscriberAlreadyActive"/> —
+    ///     the legacy guard is preserved unchanged when multi-subscriber is disabled.
+    /// </summary>
+    [Fact]
+    public async Task AttachEventSubscriber_SingleMode_SecondAttachThrowsAlreadyActive()
+    {
+        FakeWorkerClient workerClient = new();
+        GatewaySession session = CreateReadySessionWithEventStreaming(workerClient);
+
+        using IEventSubscriberLease first = session.AttachEventSubscriber(
+            allowMultipleSubscribers: false, maxSubscribers: 8);
+
+        SessionManagerException exception = Assert.Throws<SessionManagerException>(
+            () => session.AttachEventSubscriber(allowMultipleSubscribers: false, maxSubscribers: 8));
+        Assert.Equal(SessionManagerErrorCode.EventSubscriberAlreadyActive, exception.ErrorCode);
+        Assert.Equal(1, session.ActiveEventSubscriberCount);
+
+        await session.CloseAsync("test-done", CancellationToken.None);
+        await session.DisposeAsync();
+    }
+
+    /// <summary>
+    ///     Task 8. Multi-subscriber mode allows exactly <c>cap</c> concurrent external
+    ///     subscribers; the (cap+1)-th attach throws
+    ///     <see cref="SessionManagerErrorCode.EventSubscriberLimitReached"/>.
+    /// </summary>
+    [Fact]
+    public async Task AttachEventSubscriber_MultiMode_AttachesUpToCapThenThrowsLimitReached()
+    {
+        const int Cap = 4;
+        FakeWorkerClient workerClient = new();
+        GatewaySession session = CreateReadySessionWithEventStreaming(
+            workerClient,
+            allowMultipleEventSubscribers: true);
+
+        List<IEventSubscriberLease> leases = [];
+        for (int i = 0; i < Cap; i++)
+        {
+            leases.Add(session.AttachEventSubscriber(allowMultipleSubscribers: true, maxSubscribers: Cap));
+        }
+
+        Assert.Equal(Cap, session.ActiveEventSubscriberCount);
+
+        SessionManagerException exception = Assert.Throws<SessionManagerException>(
+            () => session.AttachEventSubscriber(allowMultipleSubscribers: true, maxSubscribers: Cap));
+        Assert.Equal(SessionManagerErrorCode.EventSubscriberLimitReached, exception.ErrorCode);
+        Assert.Equal(Cap, session.ActiveEventSubscriberCount);
+
+        foreach (IEventSubscriberLease lease in leases)
+        {
+            lease.Dispose();
+        }
+
+        await session.CloseAsync("test-done", CancellationToken.None);
+        await session.DisposeAsync();
+    }
+
+    /// <summary>
+    ///     Task 8. The gateway-owned INTERNAL dashboard subscriber must NOT consume cap
+    ///     budget: with the dashboard mirror running, the full cap of external subscribers is
+    ///     still attachable.
+    /// </summary>
+    [Fact]
+    public async Task AttachEventSubscriber_MultiMode_DashboardMirrorDoesNotConsumeCap()
+    {
+        const int Cap = 3;
+        FakeWorkerClient workerClient = new();
+        RecordingDashboardEventBroadcaster broadcaster = new();
+        GatewaySession session = CreateReadySessionWithEventStreaming(
+            workerClient,
+            allowMultipleEventSubscribers: true,
+            dashboardBroadcaster: broadcaster);
+
+        // The internal dashboard mirror registered on MarkReady is NOT counted as an external
+        // subscriber, so the external active count starts at zero.
+        Assert.Equal(0, session.ActiveEventSubscriberCount);
+
+        List<IEventSubscriberLease> leases = [];
+        for (int i = 0; i < Cap; i++)
+        {
+            leases.Add(session.AttachEventSubscriber(allowMultipleSubscribers: true, maxSubscribers: Cap));
+        }
+
+        Assert.Equal(Cap, session.ActiveEventSubscriberCount);
+
+        // The (cap+1)-th still fails: the dashboard mirror did not eat a slot.
+        SessionManagerException exception = Assert.Throws<SessionManagerException>(
+            () => session.AttachEventSubscriber(allowMultipleSubscribers: true, maxSubscribers: Cap));
+        Assert.Equal(SessionManagerErrorCode.EventSubscriberLimitReached, exception.ErrorCode);
+
+        foreach (IEventSubscriberLease lease in leases)
+        {
+            lease.Dispose();
+        }
+
+        await session.CloseAsync("test-done", CancellationToken.None);
+        await session.DisposeAsync();
+    }
+
+    /// <summary>
+    ///     Task 8 concurrency. Many concurrent attaches in multi-subscriber mode must never
+    ///     exceed the cap: the count-check-and-increment is atomic under <c>_syncRoot</c>, so
+    ///     exactly <c>cap</c> attaches succeed and the rest throw
+    ///     <see cref="SessionManagerErrorCode.EventSubscriberLimitReached"/>. The observed
+    ///     count never goes above the cap.
+    /// </summary>
+    [Fact]
+    public async Task AttachEventSubscriber_MultiMode_ConcurrentAttaches_NeverExceedCap()
+    {
+        const int Cap = 5;
+        const int Attempts = 32;
+        TimeSpan testTimeout = TimeSpan.FromSeconds(10);
+
+        FakeWorkerClient workerClient = new();
+        GatewaySession session = CreateReadySessionWithEventStreaming(
+            workerClient,
+            allowMultipleEventSubscribers: true);
+
+        using SemaphoreSlim gate = new(0);
+        int successCount = 0;
+        int limitReachedCount = 0;
+        int maxObservedCount = 0;
+        IEventSubscriberLease?[] leases = new IEventSubscriberLease?[Attempts];
+        Task[] tasks = new Task[Attempts];
+        for (int t = 0; t < Attempts; t++)
+        {
+            int index = t;
+            tasks[index] = Task.Run(async () =>
+            {
+                await gate.WaitAsync(testTimeout);
+                try
+                {
+                    IEventSubscriberLease lease = session.AttachEventSubscriber(
+                        allowMultipleSubscribers: true, maxSubscribers: Cap);
+                    leases[index] = lease;
+                    Interlocked.Increment(ref successCount);
+                }
+                catch (SessionManagerException exception)
+                    when (exception.ErrorCode == SessionManagerErrorCode.EventSubscriberLimitReached)
+                {
+                    Interlocked.Increment(ref limitReachedCount);
+                }
+
+                int observed = session.ActiveEventSubscriberCount;
+                int previousMax;
+                do
+                {
+                    previousMax = Volatile.Read(ref maxObservedCount);
+                    if (observed <= previousMax)
+                    {
+                        break;
+                    }
+                }
+                while (Interlocked.CompareExchange(ref maxObservedCount, observed, previousMax) != previousMax);
+            });
+        }
+
+        gate.Release(Attempts);
+        await Task.WhenAll(tasks).WaitAsync(testTimeout);
+
+        Assert.Equal(Cap, successCount);
+        Assert.Equal(Attempts - Cap, limitReachedCount);
+        Assert.Equal(Cap, session.ActiveEventSubscriberCount);
+        Assert.True(maxObservedCount <= Cap, $"Observed count {maxObservedCount} exceeded cap {Cap}.");
+
+        foreach (IEventSubscriberLease? lease in leases)
+        {
+            lease?.Dispose();
+        }
+
+        await session.CloseAsync("test-done", CancellationToken.None);
+        await session.DisposeAsync();
+    }
+
+    /// <summary>
+    ///     Task 8. Disposing a subscriber frees a cap slot so a fresh attach succeeds, and a
+    ///     double-dispose does not double-free the slot (count integrity preserved).
+    /// </summary>
+    [Fact]
+    public async Task AttachEventSubscriber_MultiMode_DisposeFreesSlotAndDoubleDisposeIsIdempotent()
+    {
+        const int Cap = 2;
+        FakeWorkerClient workerClient = new();
+        GatewaySession session = CreateReadySessionWithEventStreaming(
+            workerClient,
+            allowMultipleEventSubscribers: true);
+
+        IEventSubscriberLease a = session.AttachEventSubscriber(allowMultipleSubscribers: true, maxSubscribers: Cap);
+        IEventSubscriberLease b = session.AttachEventSubscriber(allowMultipleSubscribers: true, maxSubscribers: Cap);
+        Assert.Equal(Cap, session.ActiveEventSubscriberCount);
+
+        // At cap: next attach is rejected.
+        Assert.Throws<SessionManagerException>(
+            () => session.AttachEventSubscriber(allowMultipleSubscribers: true, maxSubscribers: Cap));
+
+        // Dispose one — and dispose it twice. The second dispose must not double-free.
+        a.Dispose();
+        a.Dispose();
+        Assert.Equal(1, session.ActiveEventSubscriberCount);
+
+        // Exactly one slot is free, so exactly one fresh attach succeeds.
+        using IEventSubscriberLease c = session.AttachEventSubscriber(allowMultipleSubscribers: true, maxSubscribers: Cap);
+        Assert.Equal(Cap, session.ActiveEventSubscriberCount);
+        Assert.Throws<SessionManagerException>(
+            () => session.AttachEventSubscriber(allowMultipleSubscribers: true, maxSubscribers: Cap));
+
+        b.Dispose();
+
+        await session.CloseAsync("test-done", CancellationToken.None);
+        await session.DisposeAsync();
+    }
+
    private static GatewaySession CreateReadySession(IWorkerClient workerClient)
    {
        GatewaySession session = new(
@@ -241,7 +457,10 @@ public sealed class GatewaySessionTests
        return session;
    }

-    private static GatewaySession CreateReadySessionWithEventStreaming(IWorkerClient workerClient)
+    private static GatewaySession CreateReadySessionWithEventStreaming(
+        IWorkerClient workerClient,
+        bool allowMultipleEventSubscribers = false,
+        IDashboardEventBroadcaster? dashboardBroadcaster = null)
    {
        GatewaySession session = new(
            sessionId: "session-test-concurrent",
@@ -262,7 +481,9 @@ public sealed class GatewaySessionTests
                new EventOptions { QueueCapacity = 8 },
                NullLogger<SessionEventDistributor>.Instance,
                TimeProvider.System,
-                new GatewayMetrics()));
+                new GatewayMetrics(),
+                dashboardBroadcaster,
+                allowMultipleEventSubscribers));
        session.AttachWorkerClient(workerClient);
        session.MarkReady();
        return session;
@@ -355,7 +355,8 @@ public sealed class SessionEventDistributorTests
                Interlocked.Increment(ref overflowCalls);
                Volatile.Write(ref observedIsOnlySubscriberValue, isOnlySubscriber);
                Volatile.Write(ref observedIsOnlySubscriberSet, 1);
-            });
+            },
+            singleSubscriberMode: false);
        await distributor.StartAsync(CancellationToken.None);

        // Slow subscriber: registered but never read, so its capacity-2 channel fills.
@@ -377,7 +378,7 @@ public sealed class SessionEventDistributorTests
            async () => await DrainUntilFaultAsync(slow.Reader));
        Assert.Equal(SessionManagerErrorCode.EventQueueOverflow, fault.ErrorCode);

-        // Two subscribers were registered at overflow time, so isOnlySubscriber is false.
+        // Multi-subscriber mode, so isOnlySubscriber is always false (Task 8 mode-gating).
        // Use Interlocked.Read / Volatile.Read so the test-thread reads are ordered after the
        // pump-thread writes, avoiding a data race by the C# memory model.
        Assert.Equal(1, Volatile.Read(ref overflowCalls));
@@ -395,15 +396,12 @@ public sealed class SessionEventDistributorTests
    public async Task SlowSubscriberOverflow_WithMultipleSubscribers_HandlerSeesIsOnlySubscriberFalse_OtherKeepsReceiving()
    {
        // Distributor-level pin for "FailFast with multiple subscribers degrades to
-        // disconnect-only (no session fault)": when the overflowing subscriber is NOT the
-        // sole subscriber, isOnlySubscriber is false, so a FailFast-wired handler must NOT
-        // fault the session. This test drives the distributor directly (without GatewaySession)
-        // with two subscribers and a FailFast-style overflow handler seam, overflows the slow
-        // one, and asserts (a) isOnlySubscriber==false, (b) the other subscriber keeps
-        // receiving, and (c) the pump keeps running — all without a GatewaySession.
-        //
-        // TODO(Task 8): add a GatewaySession-level "session stays Ready" assertion once
-        // multi-subscriber config is enabled by the Tasks 7/8 validator/guard change.
+        // disconnect-only (no session fault)": in multi-subscriber mode isOnlySubscriber is
+        // always false (Task 8 mode-gating), so a FailFast-wired handler must NOT fault the
+        // session. This test drives the distributor directly (without GatewaySession) in
+        // multi-subscriber mode with two subscribers and a FailFast-style overflow handler
+        // seam, overflows the slow one, and asserts (a) isOnlySubscriber==false, (b) the other
+        // subscriber keeps receiving, and (c) the pump keeps running.
        Channel<MxEvent> source = Channel.CreateUnbounded<MxEvent>();
        bool handlerFiredWithFalse = false;
        bool sessionFaultWouldBeCalled = false; // tracks if a FailFast path would fault
@@ -427,7 +425,8 @@ public sealed class SessionEventDistributorTests
                    // Single-subscriber: FailFast would fault the session — must not happen here.
                    Volatile.Write(ref sessionFaultWouldBeCalled, true);
                }
-            });
+            },
+            singleSubscriberMode: false);
        await distributor.StartAsync(CancellationToken.None);

        // Slow subscriber: never reads, so capacity-2 channel overflows quickly.
@@ -520,6 +519,56 @@ public sealed class SessionEventDistributorTests
            "isInternal must be true for a subscriber registered with isInternal: true.");
    }

+    [Fact]
+    public async Task SingleSubscriberMode_LoneExternalOverflow_HandlerSeesIsOnlySubscriberTrue()
+    {
+        // Task 8 mode-gating: in single-subscriber mode a lone external subscriber that
+        // overflows reports isOnlySubscriber==true, so the legacy FailFast session-fault path
+        // is preserved. The decision is gated on the fixed session mode, NOT a live count, so
+        // it is race-free.
+        Channel<MxEvent> source = Channel.CreateUnbounded<MxEvent>();
+        int observedSet = 0;
+        bool observedValue = false;
+        await using SessionEventDistributor distributor = new(
+            "session-single-sub",
+            ct => source.Reader.ReadAllAsync(ct),
+            subscriberQueueCapacity: 2,
+            replayBufferCapacity: 0,
+            replayRetentionSeconds: 0,
+            NullLogger<SessionEventDistributor>.Instance,
+            TimeProvider.System,
+            (isOnlySubscriber, _) =>
+            {
+                Volatile.Write(ref observedValue, isOnlySubscriber);
+                Volatile.Write(ref observedSet, 1);
+            },
+            singleSubscriberMode: true);
+        await distributor.StartAsync(CancellationToken.None);
+
+        using IEventSubscriberLease external = distributor.Register();
+
+        for (ulong sequence = 1; sequence <= 10; sequence++)
+        {
+            source.Writer.TryWrite(Event(sequence));
+        }
+
+        SessionManagerException fault = await Assert.ThrowsAsync<SessionManagerException>(
+            async () => await DrainUntilFaultAsync(external.Reader));
+        Assert.Equal(SessionManagerErrorCode.EventQueueOverflow, fault.ErrorCode);
+
+        await Task.Run(async () =>
+        {
+            using CancellationTokenSource cts = new(ReadTimeout);
+            while (Volatile.Read(ref observedSet) == 0)
+            {
+                await Task.Delay(10, cts.Token);
+            }
+        });
+
+        Assert.True(Volatile.Read(ref observedValue),
+            "isOnlySubscriber must be true for a lone external subscriber in single-subscriber mode.");
+    }
+
    private static async Task DrainUntilFaultAsync(ChannelReader<MxEvent> reader)
    {
        // Drains any buffered events, then surfaces the channel's completion fault (if any)
@@ -743,7 +743,7 @@ public sealed class SessionManagerTests
        GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
        DateTimeOffset now = DateTimeOffset.UtcNow;
        session.ExtendLease(now.AddSeconds(-1));
-        using IDisposable eventSubscriber = session.AttachEventSubscriber(allowMultipleSubscribers: false);
+        using IDisposable eventSubscriber = session.AttachEventSubscriber(allowMultipleSubscribers: false, maxSubscribers: 1);

        int closedCount = await manager.CloseExpiredLeasesAsync(now, CancellationToken.None);