using Opc.Ua; using Opc.Ua.Client; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; using ZB.MOM.WW.OtOpcUa.Configuration.Entities; using ZB.MOM.WW.OtOpcUa.Configuration.Enums; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Core.Authorization; namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests; /// /// Gitea #838 (follow-up to #826): the per-folder event ACL runs on the SDK's MonitoredNode2 delivery task, /// and needs the node manager's Lock on a verdict-cache miss. That delivery task drains a BOUNDED channel /// (4096, FullMode = Wait), and the synchronous NodeState.ReportEvent the node manager calls under /// Lock BLOCKS when the channel is full. So a burst of more than 4096 events reported under one hold of /// Lock, starting at a verdict-cache miss, parks the consumer on Lock while the producer waits on the /// consumer — a deadlock that also freezes every other operation needing Lock. Publishing is switched off /// on the client so nothing downstream drains either (the slow / absent consumer case). /// /// /// Since Gitea #855 condition events no longer go through that channel at all: the node manager queues them to a /// folder's items itself, per item, on the reporter's thread (as it always did for Server-object items), so the same /// flood now completes without deferring anything. The deadlock test therefore also pins that per-item delivery is in /// force (it would defer again if the bypass stopped recognising the SDK's fan-out); the ghost test produces its /// deferral with . /// public sealed class FolderEventFloodWireTests { private const string EqA = ServerEventRig.EqA; private const int FloodSize = 6000; // > the SDK's 4096-slot per-node channel /// /// Per-folder ACL on or off. The hazard does NOT need #826: with enforcement off the SDK's own /// ValidateEventRolePermissions still reaches GetNodeMetadata, which takes Lock, on the /// first event to a new item. /// [Theory] [InlineData(true)] [InlineData(false)] public async Task A_flood_beyond_the_channel_under_the_node_manager_lock_neither_deadlocks_nor_grows_unbounded(bool enforced) { var ct = TestContext.Current.CancellationToken; var rig = await ServerEventRig.StartAsync("Folder838Flood", encrypted: false, ct); var deadlocked = false; ISession? keyed = null; var reported = 0; try { rig.KeyRoles = []; rig.MaterialiseNative(); NodeAcl[] grants = [KeyEquipmentGrant(EqA)]; rig.Enforce(enforced, [ServerEventRig.Key()], grants); keyed = await rig.ConnectAsync(anonymous: false, ct); var events = new RecordedEvents(); var subscription = await RecordedEvents.SubscribeAsync(keyed, events, ct); var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(keyed, EqA)); await subscription.ApplyChangesAsync(ct); onA.Status.Error.ShouldBeNull(); // No publishing: the server-side item queue (QueueSize 1000) is the only sink, as with a stalled client. await subscription.SetPublishingModeAsync(false, ct); // #855: the folder's SDK fan-out is the one the node manager recognises and serves per item itself. rig.Nm.TryGetFolder(EqA)!.OnReportEventAsync?.Target.ShouldBeOfType(); // The whole burst under ONE hold of the node manager's Lock — the shape of any multi-event apply/projection // pass — starting from an uncached verdict (the item is new), so the delivery task needs Lock for event 1. var peakDeferred = 0; var flood = Task.Factory.StartNew(() => { lock (rig.Nm.Lock) { for (var i = 0; i < FloodSize; i++) { rig.Project(native: true, State($"f{i}")); peakDeferred = Math.Max(peakDeferred, rig.Nm.DeferredEventReports); Volatile.Write(ref reported, i + 1); } } }, ct, TaskCreationOptions.LongRunning, TaskScheduler.Default); try { await flood.WaitAsync(TimeSpan.FromSeconds(60), ct); } catch (TimeoutException) { deadlocked = true; throw new ShouldAssertException( $"reporting {FloodSize} folder events under the node manager's Lock stalled after " + $"{Volatile.Read(ref reported)} — the SDK's per-node event channel filled while its consumer waited " + "on Lock (deadlock)"); } // #855: nothing was deferred at all — every event was queued to the folder item by the reporter itself. peakDeferred.ShouldBe(0, "no notifier of ours defers: folder items are served per item, not through the SDK channel"); rig.Nm.DeferredEventReports.ShouldBe(0); // Delivery is bounded by the item's queue (QueueSize 1000, oldest discarded), not by the flood, and the // events that survive arrive in report order, ending with the last one reported. await subscription.SetPublishingModeAsync(true, ct); var last = $"f{FloodSize - 1}"; await RecordedEvents.WaitUntilAsync(() => events.Has(onA, last)); await Task.Delay(TimeSpan.FromMilliseconds(500), ct); var flooded = events.Of(onA) .Select(RecordedEvents.Message) .Where(m => m?.StartsWith('f') == true) .Select(m => int.Parse(m![1..], System.Globalization.CultureInfo.InvariantCulture)) .ToList(); flooded.Count.ShouldBeGreaterThan(0); flooded.Count.ShouldBeLessThanOrEqualTo((int)onA.Status.QueueSize, "delivery is bounded by the item's queue, not the flood"); flooded.ShouldBe(flooded.Order().ToList(), "events leave in the order they were reported"); flooded[^1].ShouldBe(FloodSize - 1); rig.Nm.DiscardedEventReports.ShouldBe(0, "nothing was deferred, so nothing was coalesced (the item queue is the bound)"); // And the node manager is live: a later event is decided by the per-folder ACL and delivered. rig.Project(native: true, State("after")); await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "after")); } finally { // A deadlocked rig cannot be disposed (its Lock is held by a blocked thread) and neither can a session on it // (CloseSession needs the server); leak both rather than hang the run. if (!deadlocked) { keyed?.Dispose(); await rig.DisposeAsync(); } } } /// /// #838 review — events deferred behind a full channel must not outlive their condition. A deploy that removes /// the condition while its events wait used to let them reach Server-object (i=2253) subscribers after the /// NodeDeleted — a ghost Retain alarm for ScadaBridge — and the folder subscribers too. Every hop of a deferred /// report now re-checks that the condition is still the live instance (#855: also after a stall at a hop, before /// the folder's items are served). /// /// Per-folder / Server-object ACL on or off (off is the production default). [Theory] [InlineData(true)] [InlineData(false)] public async Task Deferred_events_of_a_condition_removed_while_they_wait_are_never_delivered(bool enforced) { var ct = TestContext.Current.CancellationToken; await using var rig = await ServerEventRig.StartAsync("Folder838Ghost", encrypted: false, ct); rig.KeyRoles = []; rig.MaterialiseNative(); rig.Enforce(enforced, [ServerEventRig.Key()], KeyEquipmentGrant(EqA)); using var keyed = await rig.ConnectAsync(anonymous: false, ct); var events = new RecordedEvents(); var subscription = await RecordedEvents.SubscribeAsync(keyed, events, ct); var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(keyed, EqA)); var onServer = RecordedEvents.AddEventItem(subscription, ObjectIds.Server); await subscription.ApplyChangesAsync(ct); onA.Status.Error.ShouldBeNull(); onServer.Status.Error.ShouldBeNull(); await subscription.SetPublishingModeAsync(false, ct); // One Lock hold, delivery stalled at the folder: flood (the tail is deferred), then the deploy removes the condition. using (var stall = NotifierStall.Hold(rig.Nm, EqA)) { await Task.Factory.StartNew(() => { lock (rig.Nm.Lock) { for (var i = 0; i < FloodSize; i++) rig.Project(native: true, State($"f{i}")); rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the flood must actually have deferred events"); rig.Nm.RemoveAlarmConditionNode(ServerEventRig.RawAlarmPath, AddressSpaceRealm.Raw).ShouldBeTrue(); } }, ct, TaskCreationOptions.LongRunning, TaskScheduler.Default).WaitAsync(TimeSpan.FromSeconds(60), ct); stall.Release(); } await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0); rig.Nm.DiscardedEventReports.ShouldBeGreaterThan(0, "the waiting events were dropped, not delivered"); await Task.Delay(TimeSpan.FromSeconds(2), ct); await subscription.SetPublishingModeAsync(true, ct); await Task.Delay(TimeSpan.FromSeconds(2), ct); int[] Flooded(MonitoredItem item) => events.Of(item) .Select(RecordedEvents.Message) .Where(m => m?.StartsWith('f') == true) .Select(m => int.Parse(m![1..], System.Globalization.CultureInfo.InvariantCulture)) .ToArray(); // Positive control: the event reported before the stall did arrive at the Server object (synchronously, at the // condition's own hop, before the walk reached the stalled folder). Flooded(onServer).ShouldNotBeEmpty(); // None of the deferred tail — which was still waiting when the condition was removed — reached anyone. Flooded(onServer).Max().ShouldBeLessThan(FloodSize - 1, "no ghost reached the Server-object subscriber"); // Only the in-flight f0 ever reached the stalled EqA hop, and the condition was gone when the stall released: the // post-stall liveness re-check must have withheld it (#855 review 4 — "< FloodSize - 1" passed without that check). Flooded(onA).ShouldBeEmpty("no ghost reached the folder subscriber"); Flooded(onServer).ShouldBe([0], "only the event reported before the stall reached the Server object"); } private static AlarmConditionSnapshot State(string message) => new(true, false, false, true, AlarmShelvingKind.Unshelved, 700, message); private static NodeAcl KeyEquipmentGrant(string equipment) => new() { NodeAclId = $"acl-key-{equipment}", ClusterId = ServerEventRig.Cluster, Principal = ServerEventRig.KeyPrincipal, ScopeKind = NodeAclScopeKind.Equipment, ScopeId = $"eq-{equipment}", PermissionFlags = NodePermissions.ReadOnly, }; }