using Opc.Ua;
using Opc.Ua.Client;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.Authorization;
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
///
/// Gitea #838 (follow-up to #826): the per-folder event ACL runs on the SDK's MonitoredNode2 delivery task,
/// and needs the node manager's Lock on a verdict-cache miss. That delivery task drains a BOUNDED channel
/// (4096, FullMode = Wait), and the synchronous NodeState.ReportEvent the node manager calls under
/// Lock BLOCKS when the channel is full. So a burst of more than 4096 events reported under one hold of
/// Lock, starting at a verdict-cache miss, parks the consumer on Lock while the producer waits on the
/// consumer — a deadlock that also freezes every other operation needing Lock. Publishing is switched off
/// on the client so nothing downstream drains either (the slow / absent consumer case).
///
///
/// Since Gitea #855 condition events no longer go through that channel at all: the node manager queues them to a
/// folder's items itself, per item, on the reporter's thread (as it always did for Server-object items), so the same
/// flood now completes without deferring anything. The deadlock test therefore also pins that per-item delivery is in
/// force (it would defer again if the bypass stopped recognising the SDK's fan-out); the ghost test produces its
/// deferral with .
///
public sealed class FolderEventFloodWireTests
{
private const string EqA = ServerEventRig.EqA;
private const int FloodSize = 6000; // > the SDK's 4096-slot per-node channel
///
/// Per-folder ACL on or off. The hazard does NOT need #826: with enforcement off the SDK's own
/// ValidateEventRolePermissions still reaches GetNodeMetadata, which takes Lock, on the
/// first event to a new item.
///
[Theory]
[InlineData(true)]
[InlineData(false)]
public async Task A_flood_beyond_the_channel_under_the_node_manager_lock_neither_deadlocks_nor_grows_unbounded(bool enforced)
{
var ct = TestContext.Current.CancellationToken;
var rig = await ServerEventRig.StartAsync("Folder838Flood", encrypted: false, ct);
var deadlocked = false;
ISession? keyed = null;
var reported = 0;
try
{
rig.KeyRoles = [];
rig.MaterialiseNative();
NodeAcl[] grants = [KeyEquipmentGrant(EqA)];
rig.Enforce(enforced, [ServerEventRig.Key()], grants);
keyed = await rig.ConnectAsync(anonymous: false, ct);
var events = new RecordedEvents();
var subscription = await RecordedEvents.SubscribeAsync(keyed, events, ct);
var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(keyed, EqA));
await subscription.ApplyChangesAsync(ct);
onA.Status.Error.ShouldBeNull();
// No publishing: the server-side item queue (QueueSize 1000) is the only sink, as with a stalled client.
await subscription.SetPublishingModeAsync(false, ct);
// #855: the folder's SDK fan-out is the one the node manager recognises and serves per item itself.
rig.Nm.TryGetFolder(EqA)!.OnReportEventAsync?.Target.ShouldBeOfType();
// The whole burst under ONE hold of the node manager's Lock — the shape of any multi-event apply/projection
// pass — starting from an uncached verdict (the item is new), so the delivery task needs Lock for event 1.
var peakDeferred = 0;
var flood = Task.Factory.StartNew(() =>
{
lock (rig.Nm.Lock)
{
for (var i = 0; i < FloodSize; i++)
{
rig.Project(native: true, State($"f{i}"));
peakDeferred = Math.Max(peakDeferred, rig.Nm.DeferredEventReports);
Volatile.Write(ref reported, i + 1);
}
}
}, ct, TaskCreationOptions.LongRunning, TaskScheduler.Default);
try
{
await flood.WaitAsync(TimeSpan.FromSeconds(60), ct);
}
catch (TimeoutException)
{
deadlocked = true;
throw new ShouldAssertException(
$"reporting {FloodSize} folder events under the node manager's Lock stalled after "
+ $"{Volatile.Read(ref reported)} — the SDK's per-node event channel filled while its consumer waited "
+ "on Lock (deadlock)");
}
// #855: nothing was deferred at all — every event was queued to the folder item by the reporter itself.
peakDeferred.ShouldBe(0, "no notifier of ours defers: folder items are served per item, not through the SDK channel");
rig.Nm.DeferredEventReports.ShouldBe(0);
// Delivery is bounded by the item's queue (QueueSize 1000, oldest discarded), not by the flood, and the
// events that survive arrive in report order, ending with the last one reported.
await subscription.SetPublishingModeAsync(true, ct);
var last = $"f{FloodSize - 1}";
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, last));
await Task.Delay(TimeSpan.FromMilliseconds(500), ct);
var flooded = events.Of(onA)
.Select(RecordedEvents.Message)
.Where(m => m?.StartsWith('f') == true)
.Select(m => int.Parse(m![1..], System.Globalization.CultureInfo.InvariantCulture))
.ToList();
flooded.Count.ShouldBeGreaterThan(0);
flooded.Count.ShouldBeLessThanOrEqualTo((int)onA.Status.QueueSize, "delivery is bounded by the item's queue, not the flood");
flooded.ShouldBe(flooded.Order().ToList(), "events leave in the order they were reported");
flooded[^1].ShouldBe(FloodSize - 1);
rig.Nm.DiscardedEventReports.ShouldBe(0, "nothing was deferred, so nothing was coalesced (the item queue is the bound)");
// And the node manager is live: a later event is decided by the per-folder ACL and delivered.
rig.Project(native: true, State("after"));
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "after"));
}
finally
{
// A deadlocked rig cannot be disposed (its Lock is held by a blocked thread) and neither can a session on it
// (CloseSession needs the server); leak both rather than hang the run.
if (!deadlocked)
{
keyed?.Dispose();
await rig.DisposeAsync();
}
}
}
///
/// #838 review — events deferred behind a full channel must not outlive their condition. A deploy that removes
/// the condition while its events wait used to let them reach Server-object (i=2253) subscribers after the
/// NodeDeleted — a ghost Retain alarm for ScadaBridge — and the folder subscribers too. Every hop of a deferred
/// report now re-checks that the condition is still the live instance (#855: also after a stall at a hop, before
/// the folder's items are served).
///
/// Per-folder / Server-object ACL on or off (off is the production default).
[Theory]
[InlineData(true)]
[InlineData(false)]
public async Task Deferred_events_of_a_condition_removed_while_they_wait_are_never_delivered(bool enforced)
{
var ct = TestContext.Current.CancellationToken;
await using var rig = await ServerEventRig.StartAsync("Folder838Ghost", encrypted: false, ct);
rig.KeyRoles = [];
rig.MaterialiseNative();
rig.Enforce(enforced, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
using var keyed = await rig.ConnectAsync(anonymous: false, ct);
var events = new RecordedEvents();
var subscription = await RecordedEvents.SubscribeAsync(keyed, events, ct);
var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(keyed, EqA));
var onServer = RecordedEvents.AddEventItem(subscription, ObjectIds.Server);
await subscription.ApplyChangesAsync(ct);
onA.Status.Error.ShouldBeNull();
onServer.Status.Error.ShouldBeNull();
await subscription.SetPublishingModeAsync(false, ct);
// One Lock hold, delivery stalled at the folder: flood (the tail is deferred), then the deploy removes the condition.
using (var stall = NotifierStall.Hold(rig.Nm, EqA))
{
await Task.Factory.StartNew(() =>
{
lock (rig.Nm.Lock)
{
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, State($"f{i}"));
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the flood must actually have deferred events");
rig.Nm.RemoveAlarmConditionNode(ServerEventRig.RawAlarmPath, AddressSpaceRealm.Raw).ShouldBeTrue();
}
}, ct, TaskCreationOptions.LongRunning, TaskScheduler.Default).WaitAsync(TimeSpan.FromSeconds(60), ct);
stall.Release();
}
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
rig.Nm.DiscardedEventReports.ShouldBeGreaterThan(0, "the waiting events were dropped, not delivered");
await Task.Delay(TimeSpan.FromSeconds(2), ct);
await subscription.SetPublishingModeAsync(true, ct);
await Task.Delay(TimeSpan.FromSeconds(2), ct);
int[] Flooded(MonitoredItem item) => events.Of(item)
.Select(RecordedEvents.Message)
.Where(m => m?.StartsWith('f') == true)
.Select(m => int.Parse(m![1..], System.Globalization.CultureInfo.InvariantCulture))
.ToArray();
// Positive control: the event reported before the stall did arrive at the Server object (synchronously, at the
// condition's own hop, before the walk reached the stalled folder).
Flooded(onServer).ShouldNotBeEmpty();
// None of the deferred tail — which was still waiting when the condition was removed — reached anyone.
Flooded(onServer).Max().ShouldBeLessThan(FloodSize - 1, "no ghost reached the Server-object subscriber");
// Only the in-flight f0 ever reached the stalled EqA hop, and the condition was gone when the stall released: the
// post-stall liveness re-check must have withheld it (#855 review 4 — "< FloodSize - 1" passed without that check).
Flooded(onA).ShouldBeEmpty("no ghost reached the folder subscriber");
Flooded(onServer).ShouldBe([0], "only the event reported before the stall reached the Server object");
}
private static AlarmConditionSnapshot State(string message) =>
new(true, false, false, true, AlarmShelvingKind.Unshelved, 700, message);
private static NodeAcl KeyEquipmentGrant(string equipment) => new()
{
NodeAclId = $"acl-key-{equipment}",
ClusterId = ServerEventRig.Cluster,
Principal = ServerEventRig.KeyPrincipal,
ScopeKind = NodeAclScopeKind.Equipment,
ScopeId = $"eq-{equipment}",
PermissionFlags = NodePermissions.ReadOnly,
};
}