feat(sessions): multi-subscriber cap enforcement + mode-gated FailFast

2026-06-15 15:32:08 -04:00
parent 7b12eebbd1
commit ac42783e36
12 changed files with 399 additions and 81 deletions
@@ -2,10 +2,12 @@ using System.Runtime.CompilerServices;
 using Microsoft.Extensions.Logging.Abstractions;
 using ZB.MOM.WW.MxGateway.Contracts.Proto;
 using ZB.MOM.WW.MxGateway.Server.Configuration;
+using ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
 using ZB.MOM.WW.MxGateway.Server.Grpc;
 using ZB.MOM.WW.MxGateway.Server.Metrics;
 using ZB.MOM.WW.MxGateway.Server.Sessions;
 using ZB.MOM.WW.MxGateway.Server.Workers;
+using ZB.MOM.WW.MxGateway.Tests.TestSupport;

 namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Sessions;

@@ -185,7 +187,7 @@ public sealed class GatewaySessionTests
        {
            // Attach one subscriber; this increments _activeEventSubscriberCount to 1.
            IEventSubscriberLease lease = session.AttachEventSubscriber(
-                allowMultipleSubscribers: false);
+                allowMultipleSubscribers: false, maxSubscribers: 1);

            // Race Concurrency threads all calling Dispose() on the same lease.
            // Only one must actually run DetachEventSubscriber.
@@ -211,7 +213,7 @@ public sealed class GatewaySessionTests
            // Observable contract: a fresh single subscriber must now be attachable
            // (i.e., the guard _activeEventSubscriberCount > 0 is false).
            IEventSubscriberLease next = session.AttachEventSubscriber(
-                allowMultipleSubscribers: false);
+                allowMultipleSubscribers: false, maxSubscribers: 1);
            next.Dispose();
            Assert.Equal(0, session.ActiveEventSubscriberCount);
        }
@@ -220,6 +222,220 @@ public sealed class GatewaySessionTests
        await session.DisposeAsync();
    }

+    /// <summary>
+    ///     Task 8 regression. Single-subscriber mode rejects a SECOND concurrent external
+    ///     attach with <see cref="SessionManagerErrorCode.EventSubscriberAlreadyActive"/> —
+    ///     the legacy guard is preserved unchanged when multi-subscriber is disabled.
+    /// </summary>
+    [Fact]
+    public async Task AttachEventSubscriber_SingleMode_SecondAttachThrowsAlreadyActive()
+    {
+        FakeWorkerClient workerClient = new();
+        GatewaySession session = CreateReadySessionWithEventStreaming(workerClient);
+
+        using IEventSubscriberLease first = session.AttachEventSubscriber(
+            allowMultipleSubscribers: false, maxSubscribers: 8);
+
+        SessionManagerException exception = Assert.Throws<SessionManagerException>(
+            () => session.AttachEventSubscriber(allowMultipleSubscribers: false, maxSubscribers: 8));
+        Assert.Equal(SessionManagerErrorCode.EventSubscriberAlreadyActive, exception.ErrorCode);
+        Assert.Equal(1, session.ActiveEventSubscriberCount);
+
+        await session.CloseAsync("test-done", CancellationToken.None);
+        await session.DisposeAsync();
+    }
+
+    /// <summary>
+    ///     Task 8. Multi-subscriber mode allows exactly <c>cap</c> concurrent external
+    ///     subscribers; the (cap+1)-th attach throws
+    ///     <see cref="SessionManagerErrorCode.EventSubscriberLimitReached"/>.
+    /// </summary>
+    [Fact]
+    public async Task AttachEventSubscriber_MultiMode_AttachesUpToCapThenThrowsLimitReached()
+    {
+        const int Cap = 4;
+        FakeWorkerClient workerClient = new();
+        GatewaySession session = CreateReadySessionWithEventStreaming(
+            workerClient,
+            allowMultipleEventSubscribers: true);
+
+        List<IEventSubscriberLease> leases = [];
+        for (int i = 0; i < Cap; i++)
+        {
+            leases.Add(session.AttachEventSubscriber(allowMultipleSubscribers: true, maxSubscribers: Cap));
+        }
+
+        Assert.Equal(Cap, session.ActiveEventSubscriberCount);
+
+        SessionManagerException exception = Assert.Throws<SessionManagerException>(
+            () => session.AttachEventSubscriber(allowMultipleSubscribers: true, maxSubscribers: Cap));
+        Assert.Equal(SessionManagerErrorCode.EventSubscriberLimitReached, exception.ErrorCode);
+        Assert.Equal(Cap, session.ActiveEventSubscriberCount);
+
+        foreach (IEventSubscriberLease lease in leases)
+        {
+            lease.Dispose();
+        }
+
+        await session.CloseAsync("test-done", CancellationToken.None);
+        await session.DisposeAsync();
+    }
+
+    /// <summary>
+    ///     Task 8. The gateway-owned INTERNAL dashboard subscriber must NOT consume cap
+    ///     budget: with the dashboard mirror running, the full cap of external subscribers is
+    ///     still attachable.
+    /// </summary>
+    [Fact]
+    public async Task AttachEventSubscriber_MultiMode_DashboardMirrorDoesNotConsumeCap()
+    {
+        const int Cap = 3;
+        FakeWorkerClient workerClient = new();
+        RecordingDashboardEventBroadcaster broadcaster = new();
+        GatewaySession session = CreateReadySessionWithEventStreaming(
+            workerClient,
+            allowMultipleEventSubscribers: true,
+            dashboardBroadcaster: broadcaster);
+
+        // The internal dashboard mirror registered on MarkReady is NOT counted as an external
+        // subscriber, so the external active count starts at zero.
+        Assert.Equal(0, session.ActiveEventSubscriberCount);
+
+        List<IEventSubscriberLease> leases = [];
+        for (int i = 0; i < Cap; i++)
+        {
+            leases.Add(session.AttachEventSubscriber(allowMultipleSubscribers: true, maxSubscribers: Cap));
+        }
+
+        Assert.Equal(Cap, session.ActiveEventSubscriberCount);
+
+        // The (cap+1)-th still fails: the dashboard mirror did not eat a slot.
+        SessionManagerException exception = Assert.Throws<SessionManagerException>(
+            () => session.AttachEventSubscriber(allowMultipleSubscribers: true, maxSubscribers: Cap));
+        Assert.Equal(SessionManagerErrorCode.EventSubscriberLimitReached, exception.ErrorCode);
+
+        foreach (IEventSubscriberLease lease in leases)
+        {
+            lease.Dispose();
+        }
+
+        await session.CloseAsync("test-done", CancellationToken.None);
+        await session.DisposeAsync();
+    }
+
+    /// <summary>
+    ///     Task 8 concurrency. Many concurrent attaches in multi-subscriber mode must never
+    ///     exceed the cap: the count-check-and-increment is atomic under <c>_syncRoot</c>, so
+    ///     exactly <c>cap</c> attaches succeed and the rest throw
+    ///     <see cref="SessionManagerErrorCode.EventSubscriberLimitReached"/>. The observed
+    ///     count never goes above the cap.
+    /// </summary>
+    [Fact]
+    public async Task AttachEventSubscriber_MultiMode_ConcurrentAttaches_NeverExceedCap()
+    {
+        const int Cap = 5;
+        const int Attempts = 32;
+        TimeSpan testTimeout = TimeSpan.FromSeconds(10);
+
+        FakeWorkerClient workerClient = new();
+        GatewaySession session = CreateReadySessionWithEventStreaming(
+            workerClient,
+            allowMultipleEventSubscribers: true);
+
+        using SemaphoreSlim gate = new(0);
+        int successCount = 0;
+        int limitReachedCount = 0;
+        int maxObservedCount = 0;
+        IEventSubscriberLease?[] leases = new IEventSubscriberLease?[Attempts];
+        Task[] tasks = new Task[Attempts];
+        for (int t = 0; t < Attempts; t++)
+        {
+            int index = t;
+            tasks[index] = Task.Run(async () =>
+            {
+                await gate.WaitAsync(testTimeout);
+                try
+                {
+                    IEventSubscriberLease lease = session.AttachEventSubscriber(
+                        allowMultipleSubscribers: true, maxSubscribers: Cap);
+                    leases[index] = lease;
+                    Interlocked.Increment(ref successCount);
+                }
+                catch (SessionManagerException exception)
+                    when (exception.ErrorCode == SessionManagerErrorCode.EventSubscriberLimitReached)
+                {
+                    Interlocked.Increment(ref limitReachedCount);
+                }
+
+                int observed = session.ActiveEventSubscriberCount;
+                int previousMax;
+                do
+                {
+                    previousMax = Volatile.Read(ref maxObservedCount);
+                    if (observed <= previousMax)
+                    {
+                        break;
+                    }
+                }
+                while (Interlocked.CompareExchange(ref maxObservedCount, observed, previousMax) != previousMax);
+            });
+        }
+
+        gate.Release(Attempts);
+        await Task.WhenAll(tasks).WaitAsync(testTimeout);
+
+        Assert.Equal(Cap, successCount);
+        Assert.Equal(Attempts - Cap, limitReachedCount);
+        Assert.Equal(Cap, session.ActiveEventSubscriberCount);
+        Assert.True(maxObservedCount <= Cap, $"Observed count {maxObservedCount} exceeded cap {Cap}.");
+
+        foreach (IEventSubscriberLease? lease in leases)
+        {
+            lease?.Dispose();
+        }
+
+        await session.CloseAsync("test-done", CancellationToken.None);
+        await session.DisposeAsync();
+    }
+
+    /// <summary>
+    ///     Task 8. Disposing a subscriber frees a cap slot so a fresh attach succeeds, and a
+    ///     double-dispose does not double-free the slot (count integrity preserved).
+    /// </summary>
+    [Fact]
+    public async Task AttachEventSubscriber_MultiMode_DisposeFreesSlotAndDoubleDisposeIsIdempotent()
+    {
+        const int Cap = 2;
+        FakeWorkerClient workerClient = new();
+        GatewaySession session = CreateReadySessionWithEventStreaming(
+            workerClient,
+            allowMultipleEventSubscribers: true);
+
+        IEventSubscriberLease a = session.AttachEventSubscriber(allowMultipleSubscribers: true, maxSubscribers: Cap);
+        IEventSubscriberLease b = session.AttachEventSubscriber(allowMultipleSubscribers: true, maxSubscribers: Cap);
+        Assert.Equal(Cap, session.ActiveEventSubscriberCount);
+
+        // At cap: next attach is rejected.
+        Assert.Throws<SessionManagerException>(
+            () => session.AttachEventSubscriber(allowMultipleSubscribers: true, maxSubscribers: Cap));
+
+        // Dispose one — and dispose it twice. The second dispose must not double-free.
+        a.Dispose();
+        a.Dispose();
+        Assert.Equal(1, session.ActiveEventSubscriberCount);
+
+        // Exactly one slot is free, so exactly one fresh attach succeeds.
+        using IEventSubscriberLease c = session.AttachEventSubscriber(allowMultipleSubscribers: true, maxSubscribers: Cap);
+        Assert.Equal(Cap, session.ActiveEventSubscriberCount);
+        Assert.Throws<SessionManagerException>(
+            () => session.AttachEventSubscriber(allowMultipleSubscribers: true, maxSubscribers: Cap));
+
+        b.Dispose();
+
+        await session.CloseAsync("test-done", CancellationToken.None);
+        await session.DisposeAsync();
+    }
+
    private static GatewaySession CreateReadySession(IWorkerClient workerClient)
    {
        GatewaySession session = new(
@@ -241,7 +457,10 @@ public sealed class GatewaySessionTests
        return session;
    }

-    private static GatewaySession CreateReadySessionWithEventStreaming(IWorkerClient workerClient)
+    private static GatewaySession CreateReadySessionWithEventStreaming(
+        IWorkerClient workerClient,
+        bool allowMultipleEventSubscribers = false,
+        IDashboardEventBroadcaster? dashboardBroadcaster = null)
    {
        GatewaySession session = new(
            sessionId: "session-test-concurrent",
@@ -262,7 +481,9 @@ public sealed class GatewaySessionTests
                new EventOptions { QueueCapacity = 8 },
                NullLogger<SessionEventDistributor>.Instance,
                TimeProvider.System,
-                new GatewayMetrics()));
+                new GatewayMetrics(),
+                dashboardBroadcaster,
+                allowMultipleEventSubscribers));
        session.AttachWorkerClient(workerClient);
        session.MarkReady();
        return session;