using Opc.Ua;
using Opc.Ua.Client;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.Authorization;
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
///
/// Gitea #838 (follow-up to #826): the per-folder event ACL runs on the SDK's MonitoredNode2 delivery task,
/// and needs the node manager's Lock on a verdict-cache miss. That delivery task drains a BOUNDED channel
/// (4096, FullMode = Wait), and the synchronous NodeState.ReportEvent the node manager calls under
/// Lock BLOCKS when the channel is full. So a burst of more than 4096 events reported under one hold of
/// Lock, starting at a verdict-cache miss, parks the consumer on Lock while the producer waits on the
/// consumer — a deadlock that also freezes every other operation needing Lock. Publishing is switched off
/// on the client so nothing downstream drains either (the slow / absent consumer case).
///
public sealed class FolderEventFloodWireTests
{
private const string EqA = ServerEventRig.EqA;
private const int FloodSize = 6000; // > the SDK's 4096-slot per-node channel
///
/// Per-folder ACL on or off. The hazard does NOT need #826: with enforcement off the SDK's own
/// ValidateEventRolePermissions still reaches GetNodeMetadata, which takes Lock, on the
/// first event to a new item.
///
[Theory]
[InlineData(true)]
[InlineData(false)]
public async Task A_flood_beyond_the_channel_under_the_node_manager_lock_neither_deadlocks_nor_grows_unbounded(bool enforced)
{
var ct = TestContext.Current.CancellationToken;
var rig = await ServerEventRig.StartAsync("Folder838Flood", encrypted: false, ct);
var deadlocked = false;
ISession? keyed = null;
var reported = 0;
try
{
rig.KeyRoles = [];
rig.MaterialiseNative();
NodeAcl[] grants = [KeyEquipmentGrant(EqA)];
rig.Enforce(enforced, [ServerEventRig.Key()], grants);
keyed = await rig.ConnectAsync(anonymous: false, ct);
var events = new RecordedEvents();
var subscription = await RecordedEvents.SubscribeAsync(keyed, events, ct);
var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(keyed, EqA));
await subscription.ApplyChangesAsync(ct);
onA.Status.Error.ShouldBeNull();
// No publishing: the server-side item queue (QueueSize 1000) is the only sink, as with a stalled client.
await subscription.SetPublishingModeAsync(false, ct);
// The whole burst under ONE hold of the node manager's Lock — the shape of any multi-event apply/projection
// pass — starting from an uncached verdict (the item is new), so the delivery task needs Lock for event 1.
var flood = Task.Factory.StartNew(() =>
{
lock (rig.Nm.Lock)
{
for (var i = 0; i < FloodSize; i++)
{
rig.Project(native: true, State($"f{i}"));
Volatile.Write(ref reported, i + 1);
}
}
}, ct, TaskCreationOptions.LongRunning, TaskScheduler.Default);
try
{
await flood.WaitAsync(TimeSpan.FromSeconds(60), ct);
}
catch (TimeoutException)
{
deadlocked = true;
throw new ShouldAssertException(
$"reporting {FloodSize} folder events under the node manager's Lock stalled after "
+ $"{Volatile.Read(ref reported)} — the SDK's per-node event channel filled while its consumer waited "
+ "on Lock (deadlock)");
}
// The overflow was deferred rather than blocking, and drains once Lock is free.
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
// …and the SDK's channel (up to 4096 accepted events) finishes draining into the item queue, still with
// publishing off, so the count below measures the item queue and not events in transit.
await Task.Delay(TimeSpan.FromSeconds(2), ct);
// Delivery is bounded by the item's queue (QueueSize 1000, oldest discarded), not by the flood, and the
// events that survive arrive in report order, ending with the last one reported.
await subscription.SetPublishingModeAsync(true, ct);
var last = $"f{FloodSize - 1}";
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, last));
await Task.Delay(TimeSpan.FromMilliseconds(500), ct);
var flooded = events.Of(onA)
.Select(RecordedEvents.Message)
.Where(m => m?.StartsWith('f') == true)
.Select(m => int.Parse(m![1..], System.Globalization.CultureInfo.InvariantCulture))
.ToList();
flooded.Count.ShouldBeGreaterThan(0);
flooded.Count.ShouldBeLessThanOrEqualTo((int)onA.Status.QueueSize, "delivery is bounded by the item's queue, not the flood");
flooded.ShouldBe(flooded.Order().ToList(), "events leave in the order they were reported");
flooded[^1].ShouldBe(FloodSize - 1);
rig.Nm.DiscardedEventReports.ShouldBeGreaterThan(0,
"past the per-condition budget the oldest waiting events give way to the newest (coalesced)");
// And the node manager is live: a later event is decided by the per-folder ACL and delivered.
rig.Project(native: true, State("after"));
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "after"));
}
finally
{
// A deadlocked rig cannot be disposed (its Lock is held by a blocked thread) and neither can a session on it
// (CloseSession needs the server); leak both rather than hang the run.
if (!deadlocked)
{
keyed?.Dispose();
await rig.DisposeAsync();
}
}
}
///
/// #838 review — events deferred behind a full channel must not outlive their condition. A deploy that removes
/// the condition while its events wait used to let them reach Server-object (i=2253) subscribers after the
/// NodeDeleted — a ghost Retain alarm for ScadaBridge — and the folder subscribers too. Every hop of a deferred
/// report now re-checks that the condition is still the live instance.
///
/// Per-folder / Server-object ACL on or off (off is the production default).
[Theory]
[InlineData(true)]
[InlineData(false)]
public async Task Deferred_events_of_a_condition_removed_while_they_wait_are_never_delivered(bool enforced)
{
var ct = TestContext.Current.CancellationToken;
await using var rig = await ServerEventRig.StartAsync("Folder838Ghost", encrypted: false, ct);
rig.KeyRoles = [];
rig.MaterialiseNative();
rig.Enforce(enforced, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
using var keyed = await rig.ConnectAsync(anonymous: false, ct);
var events = new RecordedEvents();
var subscription = await RecordedEvents.SubscribeAsync(keyed, events, ct);
var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(keyed, EqA));
var onServer = RecordedEvents.AddEventItem(subscription, ObjectIds.Server);
await subscription.ApplyChangesAsync(ct);
onA.Status.Error.ShouldBeNull();
onServer.Status.Error.ShouldBeNull();
await subscription.SetPublishingModeAsync(false, ct);
// One Lock hold: flood past the channel (the tail is deferred), then the deploy removes the condition.
await Task.Factory.StartNew(() =>
{
lock (rig.Nm.Lock)
{
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, State($"f{i}"));
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the flood must actually have deferred events");
rig.Nm.RemoveAlarmConditionNode(ServerEventRig.RawAlarmPath, AddressSpaceRealm.Raw).ShouldBeTrue();
}
}, ct, TaskCreationOptions.LongRunning, TaskScheduler.Default).WaitAsync(TimeSpan.FromSeconds(60), ct);
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
rig.Nm.DiscardedEventReports.ShouldBeGreaterThan(0, "the waiting events were dropped, not delivered");
await Task.Delay(TimeSpan.FromSeconds(2), ct);
await subscription.SetPublishingModeAsync(true, ct);
await Task.Delay(TimeSpan.FromSeconds(2), ct);
int[] Flooded(MonitoredItem item) => events.Of(item)
.Select(RecordedEvents.Message)
.Where(m => m?.StartsWith('f') == true)
.Select(m => int.Parse(m![1..], System.Globalization.CultureInfo.InvariantCulture))
.ToArray();
// Positive control: the events reported before the channel filled did arrive (Server object: synchronously at
// the condition's own hop).
Flooded(onServer).ShouldNotBeEmpty();
// None of the deferred tail — which was still waiting when the condition was removed — reached anyone.
Flooded(onServer).Max().ShouldBeLessThan(FloodSize - 1, "no ghost reached the Server-object subscriber");
Flooded(onA).DefaultIfEmpty(-1).Max().ShouldBeLessThan(FloodSize - 1, "no ghost reached the folder subscriber");
}
private static AlarmConditionSnapshot State(string message) =>
new(true, false, false, true, AlarmShelvingKind.Unshelved, 700, message);
private static NodeAcl KeyEquipmentGrant(string equipment) => new()
{
NodeAclId = $"acl-key-{equipment}",
ClusterId = ServerEventRig.Cluster,
Principal = ServerEventRig.KeyPrincipal,
ScopeKind = NodeAclScopeKind.Equipment,
ScopeId = $"eq-{equipment}",
PermissionFlags = NodePermissions.ReadOnly,
};
}