lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderEventFloodWireTests.cs

using Opc.Ua;
using Opc.Ua.Client;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.Authorization;

namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;

/// <summary>
///     Gitea #838 (follow-up to #826): the per-folder event ACL runs on the SDK's <c>MonitoredNode2</c> delivery task,
///     and needs the node manager's <c>Lock</c> on a verdict-cache miss. That delivery task drains a BOUNDED channel
///     (4096, <c>FullMode = Wait</c>), and the synchronous <c>NodeState.ReportEvent</c> the node manager calls under
///     <c>Lock</c> BLOCKS when the channel is full. So a burst of more than 4096 events reported under one hold of
///     <c>Lock</c>, starting at a verdict-cache miss, parks the consumer on <c>Lock</c> while the producer waits on the
///     consumer — a deadlock that also freezes every other operation needing <c>Lock</c>. Publishing is switched off
///     on the client so nothing downstream drains either (the slow / absent consumer case).
/// </summary>
public sealed class FolderEventFloodWireTests
{
    private const string EqA = ServerEventRig.EqA;
    private const int FloodSize = 6000; // > the SDK's 4096-slot per-node channel

    /// <param name="enforced">
    ///     Per-folder ACL on or off. The hazard does NOT need #826: with enforcement off the SDK's own
    ///     <c>ValidateEventRolePermissions</c> still reaches <c>GetNodeMetadata</c>, which takes <c>Lock</c>, on the
    ///     first event to a new item.
    /// </param>
    [Theory]
    [InlineData(true)]
    [InlineData(false)]
    public async Task A_flood_beyond_the_channel_under_the_node_manager_lock_neither_deadlocks_nor_grows_unbounded(bool enforced)
    {
        var ct = TestContext.Current.CancellationToken;
        var rig = await ServerEventRig.StartAsync("Folder838Flood", encrypted: false, ct);
        var deadlocked = false;
        ISession? keyed = null;
        var reported = 0;
        try
        {
            rig.KeyRoles = [];
            rig.MaterialiseNative();
            NodeAcl[] grants = [KeyEquipmentGrant(EqA)];
            rig.Enforce(enforced, [ServerEventRig.Key()], grants);

            keyed = await rig.ConnectAsync(anonymous: false, ct);
            var events = new RecordedEvents();
            var subscription = await RecordedEvents.SubscribeAsync(keyed, events, ct);
            var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(keyed, EqA));
            await subscription.ApplyChangesAsync(ct);
            onA.Status.Error.ShouldBeNull();

            // No publishing: the server-side item queue (QueueSize 1000) is the only sink, as with a stalled client.
            await subscription.SetPublishingModeAsync(false, ct);

            // The whole burst under ONE hold of the node manager's Lock — the shape of any multi-event apply/projection
            // pass — starting from an uncached verdict (the item is new), so the delivery task needs Lock for event 1.
            var flood = Task.Factory.StartNew(() =>
            {
                lock (rig.Nm.Lock)
                {
                    for (var i = 0; i < FloodSize; i++)
                    {
                        rig.Project(native: true, State($"f{i}"));
                        Volatile.Write(ref reported, i + 1);
                    }
                }
            }, ct, TaskCreationOptions.LongRunning, TaskScheduler.Default);

            try
            {
                await flood.WaitAsync(TimeSpan.FromSeconds(60), ct);
            }
            catch (TimeoutException)
            {
                deadlocked = true;
                throw new ShouldAssertException(
                    $"reporting {FloodSize} folder events under the node manager's Lock stalled after "
                    + $"{Volatile.Read(ref reported)} — the SDK's per-node event channel filled while its consumer waited "
                    + "on Lock (deadlock)");
            }

            // The overflow was deferred rather than blocking, and drains once Lock is free.
            await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
            // …and the SDK's channel (up to 4096 accepted events) finishes draining into the item queue, still with
            // publishing off, so the count below measures the item queue and not events in transit.
            await Task.Delay(TimeSpan.FromSeconds(2), ct);

            // Delivery is bounded by the item's queue (QueueSize 1000, oldest discarded), not by the flood, and the
            // events that survive arrive in report order, ending with the last one reported.
            await subscription.SetPublishingModeAsync(true, ct);
            var last = $"f{FloodSize - 1}";
            await RecordedEvents.WaitUntilAsync(() => events.Has(onA, last));
            await Task.Delay(TimeSpan.FromMilliseconds(500), ct);
            var flooded = events.Of(onA)
                .Select(RecordedEvents.Message)
                .Where(m => m?.StartsWith('f') == true)
                .Select(m => int.Parse(m![1..], System.Globalization.CultureInfo.InvariantCulture))
                .ToList();
            flooded.Count.ShouldBeGreaterThan(0);
            flooded.Count.ShouldBeLessThanOrEqualTo((int)onA.Status.QueueSize, "delivery is bounded by the item's queue, not the flood");
            flooded.ShouldBe(flooded.Order().ToList(), "events leave in the order they were reported");
            flooded[^1].ShouldBe(FloodSize - 1);
            rig.Nm.DiscardedEventReports.ShouldBeGreaterThan(0,
                "past the per-condition budget the oldest waiting events give way to the newest (coalesced)");

            // And the node manager is live: a later event is decided by the per-folder ACL and delivered.
            rig.Project(native: true, State("after"));
            await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "after"));
        }
        finally
        {
            // A deadlocked rig cannot be disposed (its Lock is held by a blocked thread) and neither can a session on it
            // (CloseSession needs the server); leak both rather than hang the run.
            if (!deadlocked)
            {
                keyed?.Dispose();
                await rig.DisposeAsync();
            }
        }
    }

    /// <summary>
    ///     #838 review — events deferred behind a full channel must not outlive their condition. A deploy that removes
    ///     the condition while its events wait used to let them reach Server-object (<c>i=2253</c>) subscribers after the
    ///     NodeDeleted — a ghost Retain alarm for ScadaBridge — and the folder subscribers too. Every hop of a deferred
    ///     report now re-checks that the condition is still the live instance.
    /// </summary>
    /// <param name="enforced">Per-folder / Server-object ACL on or off (off is the production default).</param>
    [Theory]
    [InlineData(true)]
    [InlineData(false)]
    public async Task Deferred_events_of_a_condition_removed_while_they_wait_are_never_delivered(bool enforced)
    {
        var ct = TestContext.Current.CancellationToken;
        await using var rig = await ServerEventRig.StartAsync("Folder838Ghost", encrypted: false, ct);
        rig.KeyRoles = [];
        rig.MaterialiseNative();
        rig.Enforce(enforced, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));

        using var keyed = await rig.ConnectAsync(anonymous: false, ct);
        var events = new RecordedEvents();
        var subscription = await RecordedEvents.SubscribeAsync(keyed, events, ct);
        var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(keyed, EqA));
        var onServer = RecordedEvents.AddEventItem(subscription, ObjectIds.Server);
        await subscription.ApplyChangesAsync(ct);
        onA.Status.Error.ShouldBeNull();
        onServer.Status.Error.ShouldBeNull();
        await subscription.SetPublishingModeAsync(false, ct);

        // One Lock hold: flood past the channel (the tail is deferred), then the deploy removes the condition.
        await Task.Factory.StartNew(() =>
        {
            lock (rig.Nm.Lock)
            {
                for (var i = 0; i < FloodSize; i++) rig.Project(native: true, State($"f{i}"));
                rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the flood must actually have deferred events");
                rig.Nm.RemoveAlarmConditionNode(ServerEventRig.RawAlarmPath, AddressSpaceRealm.Raw).ShouldBeTrue();
            }
        }, ct, TaskCreationOptions.LongRunning, TaskScheduler.Default).WaitAsync(TimeSpan.FromSeconds(60), ct);

        await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
        rig.Nm.DiscardedEventReports.ShouldBeGreaterThan(0, "the waiting events were dropped, not delivered");
        await Task.Delay(TimeSpan.FromSeconds(2), ct);

        await subscription.SetPublishingModeAsync(true, ct);
        await Task.Delay(TimeSpan.FromSeconds(2), ct);
        int[] Flooded(MonitoredItem item) => events.Of(item)
            .Select(RecordedEvents.Message)
            .Where(m => m?.StartsWith('f') == true)
            .Select(m => int.Parse(m![1..], System.Globalization.CultureInfo.InvariantCulture))
            .ToArray();

        // Positive control: the events reported before the channel filled did arrive (Server object: synchronously at
        // the condition's own hop).
        Flooded(onServer).ShouldNotBeEmpty();
        // None of the deferred tail — which was still waiting when the condition was removed — reached anyone.
        Flooded(onServer).Max().ShouldBeLessThan(FloodSize - 1, "no ghost reached the Server-object subscriber");
        Flooded(onA).DefaultIfEmpty(-1).Max().ShouldBeLessThan(FloodSize - 1, "no ghost reached the folder subscriber");
    }

    private static AlarmConditionSnapshot State(string message) =>
        new(true, false, false, true, AlarmShelvingKind.Unshelved, 700, message);

    private static NodeAcl KeyEquipmentGrant(string equipment) => new()
    {
        NodeAclId = $"acl-key-{equipment}",
        ClusterId = ServerEventRig.Cluster,
        Principal = ServerEventRig.KeyPrincipal,
        ScopeKind = NodeAclScopeKind.Equipment,
        ScopeId = $"eq-{equipment}",
        PermissionFlags = NodePermissions.ReadOnly,
    };
}