using Opc.Ua; using Opc.Ua.Client; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; using ZB.MOM.WW.OtOpcUa.Configuration.Entities; using ZB.MOM.WW.OtOpcUa.Configuration.Enums; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Core.Authorization; namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests; /// /// Gitea #838 (follow-up to #826): the per-folder event ACL runs on the SDK's MonitoredNode2 delivery task, /// and needs the node manager's Lock on a verdict-cache miss. That delivery task drains a BOUNDED channel /// (4096, FullMode = Wait), and the synchronous NodeState.ReportEvent the node manager calls under /// Lock BLOCKS when the channel is full. So a burst of more than 4096 events reported under one hold of /// Lock, starting at a verdict-cache miss, parks the consumer on Lock while the producer waits on the /// consumer — a deadlock that also freezes every other operation needing Lock. Publishing is switched off /// on the client so nothing downstream drains either (the slow / absent consumer case). /// public sealed class FolderEventFloodWireTests { private const string EqA = ServerEventRig.EqA; private const int FloodSize = 6000; // > the SDK's 4096-slot per-node channel /// /// Per-folder ACL on or off. The hazard does NOT need #826: with enforcement off the SDK's own /// ValidateEventRolePermissions still reaches GetNodeMetadata, which takes Lock, on the /// first event to a new item. /// [Theory] [InlineData(true)] [InlineData(false)] public async Task A_flood_beyond_the_channel_under_the_node_manager_lock_neither_deadlocks_nor_grows_unbounded(bool enforced) { var ct = TestContext.Current.CancellationToken; var rig = await ServerEventRig.StartAsync("Folder838Flood", encrypted: false, ct); var deadlocked = false; ISession? keyed = null; var reported = 0; try { rig.KeyRoles = []; rig.MaterialiseNative(); NodeAcl[] grants = [KeyEquipmentGrant(EqA)]; rig.Enforce(enforced, [ServerEventRig.Key()], grants); keyed = await rig.ConnectAsync(anonymous: false, ct); var events = new RecordedEvents(); var subscription = await RecordedEvents.SubscribeAsync(keyed, events, ct); var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(keyed, EqA)); await subscription.ApplyChangesAsync(ct); onA.Status.Error.ShouldBeNull(); // No publishing: the server-side item queue (QueueSize 1000) is the only sink, as with a stalled client. await subscription.SetPublishingModeAsync(false, ct); // The whole burst under ONE hold of the node manager's Lock — the shape of any multi-event apply/projection // pass — starting from an uncached verdict (the item is new), so the delivery task needs Lock for event 1. var flood = Task.Factory.StartNew(() => { lock (rig.Nm.Lock) { for (var i = 0; i < FloodSize; i++) { rig.Project(native: true, State($"f{i}")); Volatile.Write(ref reported, i + 1); } } }, ct, TaskCreationOptions.LongRunning, TaskScheduler.Default); try { await flood.WaitAsync(TimeSpan.FromSeconds(60), ct); } catch (TimeoutException) { deadlocked = true; throw new ShouldAssertException( $"reporting {FloodSize} folder events under the node manager's Lock stalled after " + $"{Volatile.Read(ref reported)} — the SDK's per-node event channel filled while its consumer waited " + "on Lock (deadlock)"); } // The overflow was deferred rather than blocking, and drains once Lock is free. await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0); // …and the SDK's channel (up to 4096 accepted events) finishes draining into the item queue, still with // publishing off, so the count below measures the item queue and not events in transit. await Task.Delay(TimeSpan.FromSeconds(2), ct); // Delivery is bounded by the item's queue (QueueSize 1000, oldest discarded), not by the flood, and the // events that survive arrive in report order, ending with the last one reported. await subscription.SetPublishingModeAsync(true, ct); var last = $"f{FloodSize - 1}"; await RecordedEvents.WaitUntilAsync(() => events.Has(onA, last)); await Task.Delay(TimeSpan.FromMilliseconds(500), ct); var flooded = events.Of(onA) .Select(RecordedEvents.Message) .Where(m => m?.StartsWith('f') == true) .Select(m => int.Parse(m![1..], System.Globalization.CultureInfo.InvariantCulture)) .ToList(); flooded.Count.ShouldBeGreaterThan(0); flooded.Count.ShouldBeLessThanOrEqualTo((int)onA.Status.QueueSize, "delivery is bounded by the item's queue, not the flood"); flooded.ShouldBe(flooded.Order().ToList(), "events leave in the order they were reported"); flooded[^1].ShouldBe(FloodSize - 1); rig.Nm.DiscardedEventReports.ShouldBeGreaterThan(0, "past the per-condition budget the oldest waiting events give way to the newest (coalesced)"); // And the node manager is live: a later event is decided by the per-folder ACL and delivered. rig.Project(native: true, State("after")); await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "after")); } finally { // A deadlocked rig cannot be disposed (its Lock is held by a blocked thread) and neither can a session on it // (CloseSession needs the server); leak both rather than hang the run. if (!deadlocked) { keyed?.Dispose(); await rig.DisposeAsync(); } } } /// /// #838 review — events deferred behind a full channel must not outlive their condition. A deploy that removes /// the condition while its events wait used to let them reach Server-object (i=2253) subscribers after the /// NodeDeleted — a ghost Retain alarm for ScadaBridge — and the folder subscribers too. Every hop of a deferred /// report now re-checks that the condition is still the live instance. /// /// Per-folder / Server-object ACL on or off (off is the production default). [Theory] [InlineData(true)] [InlineData(false)] public async Task Deferred_events_of_a_condition_removed_while_they_wait_are_never_delivered(bool enforced) { var ct = TestContext.Current.CancellationToken; await using var rig = await ServerEventRig.StartAsync("Folder838Ghost", encrypted: false, ct); rig.KeyRoles = []; rig.MaterialiseNative(); rig.Enforce(enforced, [ServerEventRig.Key()], KeyEquipmentGrant(EqA)); using var keyed = await rig.ConnectAsync(anonymous: false, ct); var events = new RecordedEvents(); var subscription = await RecordedEvents.SubscribeAsync(keyed, events, ct); var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(keyed, EqA)); var onServer = RecordedEvents.AddEventItem(subscription, ObjectIds.Server); await subscription.ApplyChangesAsync(ct); onA.Status.Error.ShouldBeNull(); onServer.Status.Error.ShouldBeNull(); await subscription.SetPublishingModeAsync(false, ct); // One Lock hold: flood past the channel (the tail is deferred), then the deploy removes the condition. await Task.Factory.StartNew(() => { lock (rig.Nm.Lock) { for (var i = 0; i < FloodSize; i++) rig.Project(native: true, State($"f{i}")); rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the flood must actually have deferred events"); rig.Nm.RemoveAlarmConditionNode(ServerEventRig.RawAlarmPath, AddressSpaceRealm.Raw).ShouldBeTrue(); } }, ct, TaskCreationOptions.LongRunning, TaskScheduler.Default).WaitAsync(TimeSpan.FromSeconds(60), ct); await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0); rig.Nm.DiscardedEventReports.ShouldBeGreaterThan(0, "the waiting events were dropped, not delivered"); await Task.Delay(TimeSpan.FromSeconds(2), ct); await subscription.SetPublishingModeAsync(true, ct); await Task.Delay(TimeSpan.FromSeconds(2), ct); int[] Flooded(MonitoredItem item) => events.Of(item) .Select(RecordedEvents.Message) .Where(m => m?.StartsWith('f') == true) .Select(m => int.Parse(m![1..], System.Globalization.CultureInfo.InvariantCulture)) .ToArray(); // Positive control: the events reported before the channel filled did arrive (Server object: synchronously at // the condition's own hop). Flooded(onServer).ShouldNotBeEmpty(); // None of the deferred tail — which was still waiting when the condition was removed — reached anyone. Flooded(onServer).Max().ShouldBeLessThan(FloodSize - 1, "no ghost reached the Server-object subscriber"); Flooded(onA).DefaultIfEmpty(-1).Max().ShouldBeLessThan(FloodSize - 1, "no ghost reached the folder subscriber"); } private static AlarmConditionSnapshot State(string message) => new(true, false, false, true, AlarmShelvingKind.Unshelved, 700, message); private static NodeAcl KeyEquipmentGrant(string equipment) => new() { NodeAclId = $"acl-key-{equipment}", ClusterId = ServerEventRig.Cluster, Principal = ServerEventRig.KeyPrincipal, ScopeKind = NodeAclScopeKind.Equipment, ScopeId = $"eq-{equipment}", PermissionFlags = NodePermissions.ReadOnly, }; }