Files
lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/ConditionRefreshDeferralWireTests.cs
T
Joseph Doherty 1622d07f2e fix(opcua): ConditionRefresh on a Server-object item is not overtaken by older deferred condition events (#854)
While a folder subscriber's SDK channel is full, a condition's later events wait in
its #838 deferral queue and reach i=2253 items only when the drain gets to them, off
Lock. A ConditionRefresh in that window replayed the current state, then the older
deferred events followed RefreshEnd (stale state, or a condition the refresh had just
dropped re-added until the newest drained).

- Condition events are reported as ConditionEventSnapshot (an InstanceStateSnapshot
  carrying a report sequence, condition NodeId and Retain); events queued behind a
  deferred report are marked Deferred.
- Server-object refresh (ConditionRefresh and ConditionRefresh2, flag on AND off) now
  runs in RefreshServerObjectItems: watermark, "anything deferred?", the replay and its
  queueing in ONE Lock hold, so the replay is exactly the state at the cut. Per item it
  queues the replay and installs a RefreshCut under the item's gate.
- OnConditionReportEvent holds that gate across "superseded? -> QueueEvent" for
  deferred events only. Withheld from that item alone: events at/below the cut whose
  condition the replay re-sent, or Retain=true events of a condition it omitted; an
  older Retain=false event of an omitted condition is still delivered. Queues, other
  Server-object items and folder items are untouched; refresh never waits for a drain.
- Lock order Lock -> item gate -> SDK item lock, same as a synchronous report.
- Meter otopcua.opcua.condition_events.superseded.
- Folder items keep a documented residual (SDK channel backlog + memoised per-item
  verdict leave no per-event hook).

Tests: ConditionRefreshDeferralWireTests (deterministic overload by holding the folder
item's SDK lock; 6 variants fail before the fix; probe-driven gate test fails with the
gate removed; concurrent reports/refreshes/stall stress test fails before the fix).
2026-11-11 06:37:22 -05:00

431 lines
22 KiB
C#

using System.Collections;
using System.Diagnostics;
using System.Reflection;
using Opc.Ua;
using Opc.Ua.Client;
using Opc.Ua.Server;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.Core.Authorization;
using MonitoredItem = Opc.Ua.Client.MonitoredItem;
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
/// <summary>
/// Gitea #854 (from the #838 review): while a folder subscriber's event channel is full, a condition's later events
/// are deferred in its per-condition queue. A <c>ConditionRefresh</c> on a Server-object (<c>i=2253</c>) item in that
/// window delivers the condition's CURRENT state between RefreshStart and RefreshEnd — and the deferred events, all
/// older than that state, used to reach the same item AFTER RefreshEnd, so a client briefly showed (or, for a
/// condition the refresh had dropped, re-added) a stale alarm. They are now withheld from the refreshed item;
/// every other subscriber still receives them, and the refresh never waits for the stalled drain.
/// </summary>
/// <remarks>
/// The overload is produced deterministically: the folder item's server-side <c>MonitoredItem</c> lock is held, so
/// the SDK's per-node delivery task parks in <c>QueueEvent</c>, its 4096-slot channel fills, and the condition's
/// later events are deferred — without holding the node manager's <c>Lock</c>, which the refresh needs.
/// </remarks>
public sealed class ConditionRefreshDeferralWireTests
{
private const string EqA = ServerEventRig.EqA;
// > 4096 (channel) + 1 (parked in the delivery task) + 1 (the in-flight deferred write), < 1024 pending: nothing coalesces.
private const int FloodSize = 5000;
/// <param name="enforced">Per-folder / Server-object ACL on or off (off is the production default).</param>
/// <param name="finalRetained">
/// Whether the condition's state at refresh time is still Retain (so the refresh re-sends it) or has cleared
/// (so the refresh omits it and its final Retain=false event must still arrive).
/// </param>
/// <param name="refresh2">ConditionRefresh2 on the single item instead of ConditionRefresh on the subscription.</param>
[Theory]
[InlineData(false, true, false)]
[InlineData(true, true, false)]
[InlineData(false, false, false)]
[InlineData(true, false, false)]
[InlineData(false, true, true)]
[InlineData(true, false, true)]
public async Task Deferred_events_older_than_a_refresh_never_reach_the_refreshed_server_object_item_after_RefreshEnd(
bool enforced, bool finalRetained, bool refresh2)
{
var ct = TestContext.Current.CancellationToken;
await using var rig = await ServerEventRig.StartAsync("Refresh854", encrypted: false, ct);
rig.KeyRoles = [];
rig.MaterialiseNative();
rig.Enforce(enforced, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
// The folder subscriber whose channel the stall fills, on its own session so its parked item never touches the
// refreshing client's subscription.
using var folderSession = await rig.ConnectAsync(anonymous: false, ct);
var folderEvents = new RecordedEvents();
var folderSubscription = await RecordedEvents.SubscribeAsync(folderSession, folderEvents, ct);
var onA = RecordedEvents.AddEventItem(folderSubscription, ServerEventRig.UnsNodeId(folderSession, EqA));
await folderSubscription.ApplyChangesAsync(ct);
onA.Status.Error.ShouldBeNull();
using var serverSession = await rig.ConnectAsync(anonymous: false, ct);
var serverEvents = new RecordedEvents();
var serverSubscription = await RecordedEvents.SubscribeAsync(serverSession, serverEvents, ct);
var onServer = RecordedEvents.AddEventItem(serverSubscription, ObjectIds.Server);
await serverSubscription.ApplyChangesAsync(ct);
onServer.Status.Error.ShouldBeNull();
// Warm both paths (the folder item's verdict is cached on its first event).
rig.Project(native: true, Active("seed"));
await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, "seed") && serverEvents.Has(onServer, "seed"));
await folderSubscription.SetPublishingModeAsync(false, ct);
using (var stall = ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id)))
{
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the flood must actually have deferred events");
rig.Project(native: true, finalRetained ? Active("final") : Cleared("final"));
// The refresh is a service call: it must complete while the drain is still stalled.
var refresh = Stopwatch.StartNew();
if (refresh2)
{
await serverSession.CallAsync(
ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh2, ct,
serverSubscription.Id, onServer.Status.Id);
}
else
{
await serverSession.CallAsync(
ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, serverSubscription.Id);
}
refresh.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(10), "ConditionRefresh must not wait for the stalled drain");
await RecordedEvents.WaitUntilAsync(() => serverEvents.HasType(onServer, ObjectTypeIds.RefreshEndEventType));
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the deferred events were still waiting at RefreshEnd");
stall.Release();
}
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
// Positive control: a later transition is delivered (and everything deferred before it has been decided).
rig.Project(native: true, Active("after"));
await RecordedEvents.WaitUntilAsync(() => serverEvents.Has(onServer, "after"));
await Task.Delay(TimeSpan.FromMilliseconds(300), ct);
var stream = serverEvents.Of(onServer);
var start = stream.ToList().FindIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshStartEventType);
var end = stream.ToList().FindIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshEndEventType);
start.ShouldBeGreaterThanOrEqualTo(0);
end.ShouldBeGreaterThan(start);
var replayed = stream.Skip(start + 1).Take(end - start - 1).Select(RecordedEvents.Message).ToList();
var afterEnd = stream.Skip(end + 1)
.Where(f => f[RecordedEvents.AckedField].TryGetValue(out bool _))
.Select(RecordedEvents.Message)
.ToList();
if (finalRetained)
{
replayed.ShouldBe(["final"], "the refresh re-sends the condition's current state");
afterEnd.ShouldBe(["after"], "nothing older than the refreshed state may follow RefreshEnd");
}
else
{
replayed.ShouldBeEmpty("a cleared (Retain=false) condition is not part of a refresh");
// The stale Retain=true events would re-add an alarm the refresh just dropped; the final Retain=false one
// agrees with the refresh and must still arrive (a client that did not reconcile on RefreshEnd needs it).
afterEnd.ShouldBe(["final", "after"]);
stream.Last(f => RecordedEvents.Message(f) == "final")[RecordedEvents.RetainField].GetBoolean().ShouldBeFalse();
}
rig.Nm.RefreshSupersededDeliveries.ShouldBeGreaterThan(0);
// The folder subscriber did not refresh: it still receives the deferred events, in order, ending with the newest.
await folderSubscription.SetPublishingModeAsync(true, ct);
await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, "after"));
var folderFlood = folderEvents.Of(onA)
.Select(RecordedEvents.Message)
.Where(m => m?.StartsWith('f') == true && m != "final")
.Select(m => int.Parse(m![1..], System.Globalization.CultureInfo.InvariantCulture))
.ToList();
folderFlood.ShouldBe(folderFlood.Order().ToList());
folderFlood[^1].ShouldBe(FloodSize - 1, "the non-refreshing subscriber keeps the deferred tail");
folderEvents.Has(onA, "final").ShouldBeTrue();
}
/// <summary>
/// The race the item gate closes, made deterministic: a deferred event is between its supersession check and its
/// queueing to the Server-object item when the refresh arrives. The refresh must wait for that one delivery (so
/// the event lands before the replay), not slip in between and let it follow RefreshEnd.
/// </summary>
[Fact]
public async Task A_refresh_waits_for_a_deferred_delivery_already_past_its_check_so_it_lands_before_the_replay()
{
var ct = TestContext.Current.CancellationToken;
await using var rig = await ServerEventRig.StartAsync("Refresh854Gate", encrypted: false, ct);
rig.KeyRoles = [];
rig.MaterialiseNative();
rig.Enforce(false, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
using var folderSession = await rig.ConnectAsync(anonymous: false, ct);
var folderEvents = new RecordedEvents();
var folderSubscription = await RecordedEvents.SubscribeAsync(folderSession, folderEvents, ct);
var onA = RecordedEvents.AddEventItem(folderSubscription, ServerEventRig.UnsNodeId(folderSession, EqA));
await folderSubscription.ApplyChangesAsync(ct);
using var serverSession = await rig.ConnectAsync(anonymous: false, ct);
var serverEvents = new RecordedEvents();
var serverSubscription = await RecordedEvents.SubscribeAsync(serverSession, serverEvents, ct);
var onServer = RecordedEvents.AddEventItem(serverSubscription, ObjectIds.Server);
await serverSubscription.ApplyChangesAsync(ct);
rig.Project(native: true, Active("seed"));
await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, "seed") && serverEvents.Has(onServer, "seed"));
await folderSubscription.SetPublishingModeAsync(false, ct);
using var entered = new ManualResetEventSlim();
using var proceed = new ManualResetEventSlim();
var probed = 0;
rig.Nm.DeferredServerObjectDeliveryProbe = () =>
{
if (Interlocked.Increment(ref probed) != 1) return;
entered.Set();
proceed.Wait(TimeSpan.FromSeconds(30));
};
using (ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id)))
{
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
rig.Project(native: true, Active("final"));
}
// The drain resumes and parks on its first deferred delivery to the Server-object item, past the check.
entered.Wait(TimeSpan.FromSeconds(10), ct).ShouldBeTrue("the drain must reach the Server-object item");
var refresh = serverSession.CallAsync(
ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, serverSubscription.Id);
await Task.Delay(TimeSpan.FromSeconds(1), ct);
// (The SDK may answer the Call before the refresh body runs, so the replay is observed on the event stream.)
serverEvents.HasType(onServer, ObjectTypeIds.RefreshEndEventType)
.ShouldBeFalse("the refresh must wait for the delivery already inside the item's gate");
proceed.Set();
await refresh.WaitAsync(TimeSpan.FromSeconds(10), ct);
await RecordedEvents.WaitUntilAsync(() => serverEvents.HasType(onServer, ObjectTypeIds.RefreshEndEventType));
rig.Nm.DeferredServerObjectDeliveryProbe = null;
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
rig.Project(native: true, Active("after"));
await RecordedEvents.WaitUntilAsync(() => serverEvents.Has(onServer, "after"));
await Task.Delay(TimeSpan.FromMilliseconds(300), ct);
var stream = serverEvents.Of(onServer).ToList();
var start = stream.FindIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshStartEventType);
var end = stream.FindIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshEndEventType);
end.ShouldBeGreaterThan(start);
var window = stream.Skip(start + 1).Take(end - start - 1).Select(RecordedEvents.Message).ToList();
window.Count.ShouldBe(2, "the parked delivery, then the replay");
window[0]!.ShouldStartWith("f");
window[1].ShouldBe("final");
stream.Skip(end + 1)
.Where(f => f[RecordedEvents.AckedField].TryGetValue(out bool _))
.Select(RecordedEvents.Message)
.ShouldBe(["after"]);
}
/// <summary>
/// Race / stress: reports, repeated refreshes and an intermittently stalled drain all run at once. For the
/// refreshing Server-object item, no condition event delivered after a RefreshEnd may be older than the state that
/// refresh replayed, and the stream still ends on the newest state. Nothing deadlocks.
/// </summary>
/// <param name="enforced">Per-folder / Server-object ACL on or off.</param>
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Concurrent_reports_refreshes_and_drains_never_deliver_a_pre_refresh_state_after_RefreshEnd(bool enforced)
{
var ct = TestContext.Current.CancellationToken;
await using var rig = await ServerEventRig.StartAsync("Refresh854Stress", encrypted: false, ct);
rig.KeyRoles = [];
rig.MaterialiseNative();
rig.Enforce(enforced, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
using var folderSession = await rig.ConnectAsync(anonymous: false, ct);
var folderEvents = new RecordedEvents();
var folderSubscription = await RecordedEvents.SubscribeAsync(folderSession, folderEvents, ct);
var onA = RecordedEvents.AddEventItem(folderSubscription, ServerEventRig.UnsNodeId(folderSession, EqA));
await folderSubscription.ApplyChangesAsync(ct);
using var serverSession = await rig.ConnectAsync(anonymous: false, ct);
var serverEvents = new RecordedEvents();
var serverSubscription = await RecordedEvents.SubscribeAsync(serverSession, serverEvents, ct);
var onServer = RecordedEvents.AddEventItem(serverSubscription, ObjectIds.Server);
onServer.QueueSize = 100_000; // keep every event so the ordering check sees the whole stream
await serverSubscription.ApplyChangesAsync(ct);
rig.Project(native: true, Active("s0"));
await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, "s0") && serverEvents.Has(onServer, "s0"));
await folderSubscription.SetPublishingModeAsync(false, ct);
var item = FindServerItem(rig.Nm, onA.Status.Id);
using var stop = CancellationTokenSource.CreateLinkedTokenSource(ct);
var counter = 0;
var reporter = Task.Factory.StartNew(() =>
{
while (!stop.IsCancellationRequested)
{
// Bursts under one Lock hold, like an apply pass, interleaved with single reports.
lock (rig.Nm.Lock)
{
for (var i = 0; i < 500; i++) rig.Project(native: true, Active($"s{Interlocked.Increment(ref counter)}"));
}
rig.Project(native: true, Active($"s{Interlocked.Increment(ref counter)}"));
Thread.Sleep(2);
}
}, stop.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default);
var deferredSeen = 0;
var staller = Task.Factory.StartNew(() =>
{
while (!stop.IsCancellationRequested)
{
using (ItemLockStall.Hold(item))
{
Thread.Sleep(400);
if (rig.Nm.DeferredEventReports > 0) Interlocked.Increment(ref deferredSeen);
}
Thread.Sleep(100);
}
}, stop.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default);
for (var r = 0; r < 40; r++)
{
var call = r % 3 == 2
? serverSession.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh2, ct,
serverSubscription.Id, onServer.Status.Id)
: serverSession.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct,
serverSubscription.Id);
await call.WaitAsync(TimeSpan.FromSeconds(20), ct);
await Task.Delay(TimeSpan.FromMilliseconds(80), ct);
}
await stop.CancelAsync();
await reporter.WaitAsync(TimeSpan.FromSeconds(30), CancellationToken.None);
await staller.WaitAsync(TimeSpan.FromSeconds(30), CancellationToken.None);
Volatile.Read(ref deferredSeen).ShouldBeGreaterThan(1, "the stress must actually have run with deferred events");
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
var last = $"s{Volatile.Read(ref counter)}";
await folderSubscription.SetPublishingModeAsync(true, ct);
await RecordedEvents.WaitUntilAsync(() => serverEvents.Has(onServer, last));
await Task.Delay(TimeSpan.FromMilliseconds(500), ct);
// Walk the Server-object stream: after each RefreshEnd, a condition event must not be older than what that
// refresh replayed (the refresh replays the then-current state, so the floor is its replayed counter).
var floor = -1;
var replaying = false;
var replayedInThisRefresh = -1;
var violations = new List<string>();
foreach (var f in serverEvents.Of(onServer))
{
var type = f[RecordedEvents.EventTypeField].GetNodeId();
if (type == ObjectTypeIds.RefreshStartEventType)
{
replaying = true;
replayedInThisRefresh = -1;
continue;
}
if (type == ObjectTypeIds.RefreshEndEventType)
{
replaying = false;
floor = Math.Max(floor, replayedInThisRefresh);
continue;
}
if (!f[RecordedEvents.AckedField].TryGetValue(out bool _)) continue;
var n = int.Parse(RecordedEvents.Message(f)![1..], System.Globalization.CultureInfo.InvariantCulture);
if (replaying)
{
replayedInThisRefresh = n;
continue;
}
if (n < floor) violations.Add($"s{n} after a refresh that replayed s{floor}");
}
violations.ShouldBeEmpty();
rig.Nm.RefreshSupersededDeliveries.ShouldBeGreaterThan(0, "some refresh must actually have overtaken deferred events");
RecordedEvents.Message(serverEvents.Of(onServer).Last(f => f[RecordedEvents.AckedField].TryGetValue(out bool _)))
.ShouldBe(last, "the stream still converges on the newest state");
await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, last));
}
/// <summary>The server-side event item behind a client item, from the node manager's direct-item map.</summary>
private static IEventMonitoredItem FindServerItem(OtOpcUaNodeManager nm, uint id)
{
var map = (IDictionary)typeof(OtOpcUaNodeManager)
.GetField("_eventItems", BindingFlags.Instance | BindingFlags.NonPublic)!
.GetValue(nm)!;
foreach (var key in map.Keys)
{
if (key is IEventMonitoredItem item && item.Id == id) return item;
}
throw new InvalidOperationException($"no server-side event item {id}");
}
private static AlarmConditionSnapshot Active(string message) =>
new(true, false, false, true, AlarmShelvingKind.Unshelved, 700, message);
private static AlarmConditionSnapshot Cleared(string message) =>
new(false, true, true, true, AlarmShelvingKind.Unshelved, 700, message);
private static NodeAcl KeyEquipmentGrant(string equipment) => new()
{
NodeAclId = $"acl-key-{equipment}",
ClusterId = ServerEventRig.Cluster,
Principal = ServerEventRig.KeyPrincipal,
ScopeKind = NodeAclScopeKind.Equipment,
ScopeId = $"eq-{equipment}",
PermissionFlags = NodePermissions.ReadOnly,
};
/// <summary>
/// Holds a server-side monitored item's private lock on a dedicated thread, so the SDK's per-node delivery task
/// parks in <c>QueueEvent</c> for that item (a stalled subscriber) without the node manager's <c>Lock</c> held.
/// </summary>
private sealed class ItemLockStall : IDisposable
{
private readonly ManualResetEventSlim _release = new();
private readonly Thread _thread;
private ItemLockStall(IEventMonitoredItem item)
{
var gate = (Lock)typeof(Opc.Ua.Server.MonitoredItem)
.GetField("m_lock", BindingFlags.Instance | BindingFlags.NonPublic)!
.GetValue(item)!;
using var held = new ManualResetEventSlim();
_thread = new Thread(() =>
{
gate.Enter();
try
{
held.Set();
_release.Wait();
}
finally { gate.Exit(); }
}) { IsBackground = true };
_thread.Start();
held.Wait();
}
public static ItemLockStall Hold(IEventMonitoredItem item) => new(item);
public void Release() => _release.Set();
public void Dispose()
{
_release.Set();
_thread.Join();
_release.Dispose();
}
}
}