using System.Collections; using System.Diagnostics; using System.Reflection; using Opc.Ua; using Opc.Ua.Client; using Opc.Ua.Server; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; using ZB.MOM.WW.OtOpcUa.Configuration.Entities; using ZB.MOM.WW.OtOpcUa.Configuration.Enums; using ZB.MOM.WW.OtOpcUa.Core.Authorization; using MonitoredItem = Opc.Ua.Client.MonitoredItem; namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests; /// /// Gitea #854 (from the #838 review): while a folder subscriber's event channel is full, a condition's later events /// are deferred in its per-condition queue. A ConditionRefresh on a Server-object (i=2253) item in that /// window delivers the condition's CURRENT state between RefreshStart and RefreshEnd — and the deferred events, all /// older than that state, used to reach the same item AFTER RefreshEnd, so a client briefly showed (or, for a /// condition the refresh had dropped, re-added) a stale alarm. They are now withheld from the refreshed item; /// every other subscriber still receives them, and the refresh never waits for the stalled drain. /// /// /// The overload is produced deterministically: the folder item's server-side MonitoredItem lock is held, so /// the SDK's per-node delivery task parks in QueueEvent, its 4096-slot channel fills, and the condition's /// later events are deferred — without holding the node manager's Lock, which the refresh needs. /// public sealed class ConditionRefreshDeferralWireTests { private const string EqA = ServerEventRig.EqA; // > 4096 (channel) + 1 (parked in the delivery task) + 1 (the in-flight deferred write), < 1024 pending: nothing coalesces. private const int FloodSize = 5000; /// Per-folder / Server-object ACL on or off (off is the production default). /// /// Whether the condition's state at refresh time is still Retain (so the refresh re-sends it) or has cleared /// (so the refresh omits it and its final Retain=false event must still arrive). /// /// ConditionRefresh2 on the single item instead of ConditionRefresh on the subscription. [Theory] [InlineData(false, true, false)] [InlineData(true, true, false)] [InlineData(false, false, false)] [InlineData(true, false, false)] [InlineData(false, true, true)] [InlineData(true, false, true)] public async Task Deferred_events_older_than_a_refresh_never_reach_the_refreshed_server_object_item_after_RefreshEnd( bool enforced, bool finalRetained, bool refresh2) { var ct = TestContext.Current.CancellationToken; await using var rig = await ServerEventRig.StartAsync("Refresh854", encrypted: false, ct); rig.KeyRoles = []; rig.MaterialiseNative(); rig.Enforce(enforced, [ServerEventRig.Key()], KeyEquipmentGrant(EqA)); // The folder subscriber whose channel the stall fills, on its own session so its parked item never touches the // refreshing client's subscription. using var folderSession = await rig.ConnectAsync(anonymous: false, ct); var folderEvents = new RecordedEvents(); var folderSubscription = await RecordedEvents.SubscribeAsync(folderSession, folderEvents, ct); var onA = RecordedEvents.AddEventItem(folderSubscription, ServerEventRig.UnsNodeId(folderSession, EqA)); await folderSubscription.ApplyChangesAsync(ct); onA.Status.Error.ShouldBeNull(); using var serverSession = await rig.ConnectAsync(anonymous: false, ct); var serverEvents = new RecordedEvents(); var serverSubscription = await RecordedEvents.SubscribeAsync(serverSession, serverEvents, ct); var onServer = RecordedEvents.AddEventItem(serverSubscription, ObjectIds.Server); await serverSubscription.ApplyChangesAsync(ct); onServer.Status.Error.ShouldBeNull(); // Warm both paths (the folder item's verdict is cached on its first event). rig.Project(native: true, Active("seed")); await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, "seed") && serverEvents.Has(onServer, "seed")); await folderSubscription.SetPublishingModeAsync(false, ct); using (var stall = ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id))) { for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}")); rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the flood must actually have deferred events"); rig.Project(native: true, finalRetained ? Active("final") : Cleared("final")); // The refresh is a service call: it must complete while the drain is still stalled. var refresh = Stopwatch.StartNew(); if (refresh2) { await serverSession.CallAsync( ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh2, ct, serverSubscription.Id, onServer.Status.Id); } else { await serverSession.CallAsync( ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, serverSubscription.Id); } refresh.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(10), "ConditionRefresh must not wait for the stalled drain"); await RecordedEvents.WaitUntilAsync(() => serverEvents.HasType(onServer, ObjectTypeIds.RefreshEndEventType)); rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the deferred events were still waiting at RefreshEnd"); stall.Release(); } await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0); // Positive control: a later transition is delivered (and everything deferred before it has been decided). rig.Project(native: true, Active("after")); await RecordedEvents.WaitUntilAsync(() => serverEvents.Has(onServer, "after")); await Task.Delay(TimeSpan.FromMilliseconds(300), ct); var stream = serverEvents.Of(onServer); var start = stream.ToList().FindIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshStartEventType); var end = stream.ToList().FindIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshEndEventType); start.ShouldBeGreaterThanOrEqualTo(0); end.ShouldBeGreaterThan(start); var replayed = stream.Skip(start + 1).Take(end - start - 1).Select(RecordedEvents.Message).ToList(); var afterEnd = stream.Skip(end + 1) .Where(f => f[RecordedEvents.AckedField].TryGetValue(out bool _)) .Select(RecordedEvents.Message) .ToList(); if (finalRetained) { replayed.ShouldBe(["final"], "the refresh re-sends the condition's current state"); afterEnd.ShouldBe(["after"], "nothing older than the refreshed state may follow RefreshEnd"); } else { replayed.ShouldBeEmpty("a cleared (Retain=false) condition is not part of a refresh"); // The stale Retain=true events would re-add an alarm the refresh just dropped; the final Retain=false one // agrees with the refresh and must still arrive (a client that did not reconcile on RefreshEnd needs it). afterEnd.ShouldBe(["final", "after"]); stream.Last(f => RecordedEvents.Message(f) == "final")[RecordedEvents.RetainField].GetBoolean().ShouldBeFalse(); } rig.Nm.RefreshSupersededDeliveries.ShouldBeGreaterThan(0); // The folder subscriber did not refresh: it still receives the deferred events, in order, ending with the newest. await folderSubscription.SetPublishingModeAsync(true, ct); await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, "after")); var folderFlood = folderEvents.Of(onA) .Select(RecordedEvents.Message) .Where(m => m?.StartsWith('f') == true && m != "final") .Select(m => int.Parse(m![1..], System.Globalization.CultureInfo.InvariantCulture)) .ToList(); folderFlood.ShouldBe(folderFlood.Order().ToList()); folderFlood[^1].ShouldBe(FloodSize - 1, "the non-refreshing subscriber keeps the deferred tail"); folderEvents.Has(onA, "final").ShouldBeTrue(); } /// /// The race the item gate closes, made deterministic: a deferred event is between its supersession check and its /// queueing to the Server-object item when the refresh arrives. The refresh must wait for that one delivery (so /// the event lands before the replay), not slip in between and let it follow RefreshEnd. /// [Fact] public async Task A_refresh_waits_for_a_deferred_delivery_already_past_its_check_so_it_lands_before_the_replay() { var ct = TestContext.Current.CancellationToken; await using var rig = await ServerEventRig.StartAsync("Refresh854Gate", encrypted: false, ct); rig.KeyRoles = []; rig.MaterialiseNative(); rig.Enforce(false, [ServerEventRig.Key()], KeyEquipmentGrant(EqA)); using var folderSession = await rig.ConnectAsync(anonymous: false, ct); var folderEvents = new RecordedEvents(); var folderSubscription = await RecordedEvents.SubscribeAsync(folderSession, folderEvents, ct); var onA = RecordedEvents.AddEventItem(folderSubscription, ServerEventRig.UnsNodeId(folderSession, EqA)); await folderSubscription.ApplyChangesAsync(ct); using var serverSession = await rig.ConnectAsync(anonymous: false, ct); var serverEvents = new RecordedEvents(); var serverSubscription = await RecordedEvents.SubscribeAsync(serverSession, serverEvents, ct); var onServer = RecordedEvents.AddEventItem(serverSubscription, ObjectIds.Server); await serverSubscription.ApplyChangesAsync(ct); rig.Project(native: true, Active("seed")); await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, "seed") && serverEvents.Has(onServer, "seed")); await folderSubscription.SetPublishingModeAsync(false, ct); using var entered = new ManualResetEventSlim(); using var proceed = new ManualResetEventSlim(); var probed = 0; rig.Nm.DeferredServerObjectDeliveryProbe = () => { if (Interlocked.Increment(ref probed) != 1) return; entered.Set(); proceed.Wait(TimeSpan.FromSeconds(30)); }; using (ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id))) { for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}")); rig.Project(native: true, Active("final")); } // The drain resumes and parks on its first deferred delivery to the Server-object item, past the check. entered.Wait(TimeSpan.FromSeconds(10), ct).ShouldBeTrue("the drain must reach the Server-object item"); var refresh = serverSession.CallAsync( ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, serverSubscription.Id); await Task.Delay(TimeSpan.FromSeconds(1), ct); // (The SDK may answer the Call before the refresh body runs, so the replay is observed on the event stream.) serverEvents.HasType(onServer, ObjectTypeIds.RefreshEndEventType) .ShouldBeFalse("the refresh must wait for the delivery already inside the item's gate"); proceed.Set(); await refresh.WaitAsync(TimeSpan.FromSeconds(10), ct); await RecordedEvents.WaitUntilAsync(() => serverEvents.HasType(onServer, ObjectTypeIds.RefreshEndEventType)); rig.Nm.DeferredServerObjectDeliveryProbe = null; await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0); rig.Project(native: true, Active("after")); await RecordedEvents.WaitUntilAsync(() => serverEvents.Has(onServer, "after")); await Task.Delay(TimeSpan.FromMilliseconds(300), ct); var stream = serverEvents.Of(onServer).ToList(); var start = stream.FindIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshStartEventType); var end = stream.FindIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshEndEventType); end.ShouldBeGreaterThan(start); var window = stream.Skip(start + 1).Take(end - start - 1).Select(RecordedEvents.Message).ToList(); window.Count.ShouldBe(2, "the parked delivery, then the replay"); window[0]!.ShouldStartWith("f"); window[1].ShouldBe("final"); stream.Skip(end + 1) .Where(f => f[RecordedEvents.AckedField].TryGetValue(out bool _)) .Select(RecordedEvents.Message) .ShouldBe(["after"]); } /// /// Race / stress: reports, repeated refreshes and an intermittently stalled drain all run at once. For the /// refreshing Server-object item, no condition event delivered after a RefreshEnd may be older than the state that /// refresh replayed, and the stream still ends on the newest state. Nothing deadlocks. /// /// Per-folder / Server-object ACL on or off. [Theory] [InlineData(false)] [InlineData(true)] public async Task Concurrent_reports_refreshes_and_drains_never_deliver_a_pre_refresh_state_after_RefreshEnd(bool enforced) { var ct = TestContext.Current.CancellationToken; await using var rig = await ServerEventRig.StartAsync("Refresh854Stress", encrypted: false, ct); rig.KeyRoles = []; rig.MaterialiseNative(); rig.Enforce(enforced, [ServerEventRig.Key()], KeyEquipmentGrant(EqA)); using var folderSession = await rig.ConnectAsync(anonymous: false, ct); var folderEvents = new RecordedEvents(); var folderSubscription = await RecordedEvents.SubscribeAsync(folderSession, folderEvents, ct); var onA = RecordedEvents.AddEventItem(folderSubscription, ServerEventRig.UnsNodeId(folderSession, EqA)); await folderSubscription.ApplyChangesAsync(ct); using var serverSession = await rig.ConnectAsync(anonymous: false, ct); var serverEvents = new RecordedEvents(); var serverSubscription = await RecordedEvents.SubscribeAsync(serverSession, serverEvents, ct); var onServer = RecordedEvents.AddEventItem(serverSubscription, ObjectIds.Server); onServer.QueueSize = 100_000; // keep every event so the ordering check sees the whole stream await serverSubscription.ApplyChangesAsync(ct); rig.Project(native: true, Active("s0")); await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, "s0") && serverEvents.Has(onServer, "s0")); await folderSubscription.SetPublishingModeAsync(false, ct); var item = FindServerItem(rig.Nm, onA.Status.Id); using var stop = CancellationTokenSource.CreateLinkedTokenSource(ct); var counter = 0; var reporter = Task.Factory.StartNew(() => { while (!stop.IsCancellationRequested) { // Bursts under one Lock hold, like an apply pass, interleaved with single reports. lock (rig.Nm.Lock) { for (var i = 0; i < 500; i++) rig.Project(native: true, Active($"s{Interlocked.Increment(ref counter)}")); } rig.Project(native: true, Active($"s{Interlocked.Increment(ref counter)}")); Thread.Sleep(2); } }, stop.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default); var deferredSeen = 0; var staller = Task.Factory.StartNew(() => { while (!stop.IsCancellationRequested) { using (ItemLockStall.Hold(item)) { Thread.Sleep(400); if (rig.Nm.DeferredEventReports > 0) Interlocked.Increment(ref deferredSeen); } Thread.Sleep(100); } }, stop.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default); for (var r = 0; r < 40; r++) { var call = r % 3 == 2 ? serverSession.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh2, ct, serverSubscription.Id, onServer.Status.Id) : serverSession.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, serverSubscription.Id); await call.WaitAsync(TimeSpan.FromSeconds(20), ct); await Task.Delay(TimeSpan.FromMilliseconds(80), ct); } await stop.CancelAsync(); await reporter.WaitAsync(TimeSpan.FromSeconds(30), CancellationToken.None); await staller.WaitAsync(TimeSpan.FromSeconds(30), CancellationToken.None); Volatile.Read(ref deferredSeen).ShouldBeGreaterThan(1, "the stress must actually have run with deferred events"); await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0); var last = $"s{Volatile.Read(ref counter)}"; await folderSubscription.SetPublishingModeAsync(true, ct); await RecordedEvents.WaitUntilAsync(() => serverEvents.Has(onServer, last)); await Task.Delay(TimeSpan.FromMilliseconds(500), ct); // Walk the Server-object stream: after each RefreshEnd, a condition event must not be older than what that // refresh replayed (the refresh replays the then-current state, so the floor is its replayed counter). var floor = -1; var replaying = false; var replayedInThisRefresh = -1; var violations = new List(); foreach (var f in serverEvents.Of(onServer)) { var type = f[RecordedEvents.EventTypeField].GetNodeId(); if (type == ObjectTypeIds.RefreshStartEventType) { replaying = true; replayedInThisRefresh = -1; continue; } if (type == ObjectTypeIds.RefreshEndEventType) { replaying = false; floor = Math.Max(floor, replayedInThisRefresh); continue; } if (!f[RecordedEvents.AckedField].TryGetValue(out bool _)) continue; var n = int.Parse(RecordedEvents.Message(f)![1..], System.Globalization.CultureInfo.InvariantCulture); if (replaying) { replayedInThisRefresh = n; continue; } if (n < floor) violations.Add($"s{n} after a refresh that replayed s{floor}"); } violations.ShouldBeEmpty(); rig.Nm.RefreshSupersededDeliveries.ShouldBeGreaterThan(0, "some refresh must actually have overtaken deferred events"); RecordedEvents.Message(serverEvents.Of(onServer).Last(f => f[RecordedEvents.AckedField].TryGetValue(out bool _))) .ShouldBe(last, "the stream still converges on the newest state"); await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, last)); } /// The server-side event item behind a client item, from the node manager's direct-item map. private static IEventMonitoredItem FindServerItem(OtOpcUaNodeManager nm, uint id) { var map = (IDictionary)typeof(OtOpcUaNodeManager) .GetField("_eventItems", BindingFlags.Instance | BindingFlags.NonPublic)! .GetValue(nm)!; foreach (var key in map.Keys) { if (key is IEventMonitoredItem item && item.Id == id) return item; } throw new InvalidOperationException($"no server-side event item {id}"); } private static AlarmConditionSnapshot Active(string message) => new(true, false, false, true, AlarmShelvingKind.Unshelved, 700, message); private static AlarmConditionSnapshot Cleared(string message) => new(false, true, true, true, AlarmShelvingKind.Unshelved, 700, message); private static NodeAcl KeyEquipmentGrant(string equipment) => new() { NodeAclId = $"acl-key-{equipment}", ClusterId = ServerEventRig.Cluster, Principal = ServerEventRig.KeyPrincipal, ScopeKind = NodeAclScopeKind.Equipment, ScopeId = $"eq-{equipment}", PermissionFlags = NodePermissions.ReadOnly, }; /// /// Holds a server-side monitored item's private lock on a dedicated thread, so the SDK's per-node delivery task /// parks in QueueEvent for that item (a stalled subscriber) without the node manager's Lock held. /// private sealed class ItemLockStall : IDisposable { private readonly ManualResetEventSlim _release = new(); private readonly Thread _thread; private ItemLockStall(IEventMonitoredItem item) { var gate = (Lock)typeof(Opc.Ua.Server.MonitoredItem) .GetField("m_lock", BindingFlags.Instance | BindingFlags.NonPublic)! .GetValue(item)!; using var held = new ManualResetEventSlim(); _thread = new Thread(() => { gate.Enter(); try { held.Set(); _release.Wait(); } finally { gate.Exit(); } }) { IsBackground = true }; _thread.Start(); held.Wait(); } public static ItemLockStall Hold(IEventMonitoredItem item) => new(item); public void Release() => _release.Set(); public void Dispose() { _release.Set(); _thread.Join(); _release.Dispose(); } } }