Merge #855 folder event items: per-item delivery under the node manager, ConditionRefresh cut matches the Server object

Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
Joseph Doherty
2026-11-11 13:07:21 -05:00
10 changed files with 1654 additions and 228 deletions
@@ -24,16 +24,18 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
/// every other subscriber still receives them, and the refresh never waits for the stalled drain.
/// </summary>
/// <remarks>
/// The overload is produced deterministically: the folder item's server-side <c>MonitoredItem</c> lock is held, so
/// the SDK's per-node delivery task parks in <c>QueueEvent</c>, its 4096-slot channel fills, and the condition's
/// later events are deferred — without holding the node manager's <c>Lock</c>, which the refresh needs.
/// The overload is produced deterministically: delivery is stalled at the EqA folder (<see cref="NotifierStall"/>), so the
/// condition's walk suspends there and its later events are deferred — without holding the node manager's <c>Lock</c>,
/// which the refresh needs. (Before Gitea #855 this held the folder item's SDK lock so the folder's 4096-slot channel
/// filled; folder items are now queued per item by the reporter, and no notifier of ours defers on its own.) The
/// folder-item twin of these tests is <see cref="FolderConditionRefreshWireTests"/>.
/// </remarks>
public sealed class ConditionRefreshDeferralWireTests
{
private const string EqA = ServerEventRig.EqA;
// > 4096 (channel) + 1 (parked in the delivery task) + 1 (the in-flight deferred write), < 1024 pending: nothing coalesces.
private const int FloodSize = 5000;
// Every event after the stalled one is deferred; < 1024 pending, so nothing coalesces.
private const int FloodSize = 500;
/// <param name="enforced">Per-folder / Server-object ACL on or off (off is the production default).</param>
/// <param name="finalRetained">
@@ -84,7 +86,7 @@ public sealed class ConditionRefreshDeferralWireTests
await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, "seed") && serverEvents.Has(onServer, "seed"));
await folderSubscription.SetPublishingModeAsync(false, ct);
using (var stall = ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id)))
using (var stall = NotifierStall.Hold(rig.Nm, EqA))
{
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the flood must actually have deferred events");
@@ -200,7 +202,7 @@ public sealed class ConditionRefreshDeferralWireTests
serverEvents.Has(onServer, "seed").ShouldBeFalse("the key holds no grant yet");
await folderSubscription.SetPublishingModeAsync(false, ct);
using (var stall = ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id)))
using (var stall = NotifierStall.Hold(rig.Nm, EqA))
{
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
rig.Project(native: true, Active("final"));
@@ -271,7 +273,7 @@ public sealed class ConditionRefreshDeferralWireTests
rig.Sink.WriteAlarmCondition(Free, Active("x2"), DateTime.UtcNow, AddressSpaceRealm.Raw);
};
using (var stall = ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id)))
using (var stall = NotifierStall.Hold(rig.Nm, EqA))
{
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the cut path runs only while something is deferred");
@@ -345,7 +347,7 @@ public sealed class ConditionRefreshDeferralWireTests
proceed.Wait(TimeSpan.FromSeconds(30));
};
using (ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id)))
using (NotifierStall.Hold(rig.Nm, EqA))
{
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
rig.Project(native: true, Active("final"));
@@ -416,7 +418,6 @@ public sealed class ConditionRefreshDeferralWireTests
rig.Project(native: true, Active("s0"));
await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, "s0") && serverEvents.Has(onServer, "s0"));
await folderSubscription.SetPublishingModeAsync(false, ct);
var item = FindServerItem(rig.Nm, onA.Status.Id);
using var stop = CancellationTokenSource.CreateLinkedTokenSource(ct);
var counter = 0;
@@ -440,7 +441,7 @@ public sealed class ConditionRefreshDeferralWireTests
{
while (!stop.IsCancellationRequested)
{
using (ItemLockStall.Hold(item))
using (NotifierStall.Hold(rig.Nm, EqA))
{
Thread.Sleep(400);
if (rig.Nm.DeferredEventReports > 0) Interlocked.Increment(ref deferredSeen);
@@ -568,7 +569,7 @@ public sealed class ConditionRefreshDeferralWireTests
await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, "seed") && serverEvents.Has(onServer, "seed"));
await folderSubscription.SetPublishingModeAsync(false, ct);
var stall = deferring ? ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id)) : null;
var stall = deferring ? NotifierStall.Hold(rig.Nm, EqA) : null;
try
{
if (deferring)
@@ -622,7 +623,7 @@ public sealed class ConditionRefreshDeferralWireTests
}
/// <summary>The server-side Server-object (all-events) item behind a client item.</summary>
private static IEventMonitoredItem FindAllEventsItem(OtOpcUaNodeManager nm, uint id)
internal static IEventMonitoredItem FindAllEventsItem(OtOpcUaNodeManager nm, uint id)
{
var map = (IDictionary)typeof(OtOpcUaNodeManager)
.GetField("_allEventsItems", BindingFlags.Instance | BindingFlags.NonPublic)!
@@ -636,7 +637,7 @@ public sealed class ConditionRefreshDeferralWireTests
}
/// <summary>The server-side event item behind a client item, from the node manager's direct-item map.</summary>
private static IEventMonitoredItem FindServerItem(OtOpcUaNodeManager nm, uint id)
internal static IEventMonitoredItem FindServerItem(OtOpcUaNodeManager nm, uint id)
{
var map = (IDictionary)typeof(OtOpcUaNodeManager)
.GetField("_eventItems", BindingFlags.Instance | BindingFlags.NonPublic)!
@@ -664,45 +665,4 @@ public sealed class ConditionRefreshDeferralWireTests
ScopeId = $"eq-{equipment}",
PermissionFlags = NodePermissions.ReadOnly,
};
/// <summary>
/// Holds a server-side monitored item's private lock on a dedicated thread, so the SDK's per-node delivery task
/// parks in <c>QueueEvent</c> for that item (a stalled subscriber) without the node manager's <c>Lock</c> held.
/// </summary>
private sealed class ItemLockStall : IDisposable
{
private readonly ManualResetEventSlim _release = new();
private readonly Thread _thread;
private ItemLockStall(IEventMonitoredItem item)
{
var gate = (Lock)typeof(Opc.Ua.Server.MonitoredItem)
.GetField("m_lock", BindingFlags.Instance | BindingFlags.NonPublic)!
.GetValue(item)!;
using var held = new ManualResetEventSlim();
_thread = new Thread(() =>
{
gate.Enter();
try
{
held.Set();
_release.Wait();
}
finally { gate.Exit(); }
}) { IsBackground = true };
_thread.Start();
held.Wait();
}
public static ItemLockStall Hold(IEventMonitoredItem item) => new(item);
public void Release() => _release.Set();
public void Dispose()
{
_release.Set();
_thread.Join();
_release.Dispose();
}
}
}
@@ -27,13 +27,16 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
/// <item>no ACL-denied event reaches an item during the overflow, including deferred events whose grant was
/// revoked while they waited.</item>
/// </list>
/// Since Gitea #855 the folder items are served per item by the reporter (as the Server-object items always were),
/// so the flood no longer defers at all — the first test pins that — and the revocation test stalls the folder's
/// delivery (<see cref="NotifierStall"/>) to get a deferred backlog.
/// </summary>
public sealed class EventQueueOverflowAclWireTests
{
private const string EqA = ServerEventRig.EqA;
private const string EqB = ServerEventRig.EqB;
// Native events per flood: past the SDK's 4096-slot per-notifier channel, so the native condition defers.
// Native events per flood: past the SDK's 4096-slot per-notifier channel (which deferred them before #855).
private const int PerCondition = 6000;
/// <param name="singleHold">
@@ -150,13 +153,12 @@ public sealed class EventQueueOverflowAclWireTests
readLatencies.Max().ShouldBeLessThan(TimeSpan.FromSeconds(5), "a Read must not queue behind the whole flood");
(await TimedReadAsync(anonymous, eqA, ct)).ShouldBeLessThan(TimeSpan.FromSeconds(2), "the node manager is free afterwards");
// Memory: the node manager's own overflow store stayed within its per-condition budget (queued + in flight)…
Volatile.Read(ref peakDeferred).ShouldBeLessThanOrEqualTo(
2 * (OtOpcUaNodeManager.MaxDeferredEventsPerCondition + 1), "deferral is bounded per condition");
if (singleHold) rig.Nm.DiscardedEventReports.ShouldBeGreaterThan(0, "the native tail overflowed and coalesced");
// Memory: the node manager held nothing of its own for the overflow (#855: no notifier of ours defers, so the
// item queues are the only store, bounded below)…
Volatile.Read(ref peakDeferred).ShouldBe(0, "folder items are served per item by the reporter: nothing defers");
rig.Nm.DiscardedEventReports.ShouldBe(0);
// …let the SDK channel finish draining into the (still unpublished) item queues, then publish.
await Task.Delay(TimeSpan.FromSeconds(2), ct);
// …then publish the (still unpublished) item queues.
await keySub.SetPublishingModeAsync(true, ct);
var lastNative = $"n{PerCondition - 1}";
await RecordedEvents.WaitUntilAsync(() => keyEvents.Has(onB, lastNative) && keyEvents.Has(keyServer, lastNative));
@@ -165,10 +167,9 @@ public sealed class EventQueueOverflowAclWireTests
foreach (var item in new[] { onB, keyServer })
{
var received = keyEvents.Of(item);
// ACL: not one event of the denied (EqA-owned) scripted alarm, overflow or not. (Under a single hold the
// deferred native tail drains last and evicts every synchronously-queued event from the 1000-slot queue, so
// there this check cannot see a leak; the "after-s" leg below is the falsifier for that mode — verified by
// running with enforcement off: chunked fails here, single-hold fails at "after-s".)
// ACL: not one event of the denied (EqA-owned) scripted alarm, overflow or not. (The 1000-slot queue keeps
// only the last events of the flood, which interleave n/s, so a leak shows here in both modes; the
// "after-s" leg below is a second falsifier.)
received.Where(f => RecordedEvents.Message(f)?.StartsWith('s') == true)
.ShouldBeEmpty($"{item.DisplayName}: an ACL-denied event leaked during the overflow");
@@ -211,7 +212,7 @@ public sealed class EventQueueOverflowAclWireTests
}
/// <summary>
/// A grant revoked while a deferred backlog waits behind a full notifier channel: the deferred events are decided
/// A grant revoked while a deferred backlog waits behind a stalled notifier: the deferred events are decided
/// at delivery against the CURRENT generation, so none of them — nor any event reported after the revocation —
/// reaches the revoked session's folder or Server-object item, while a still-granted session sharing the same
/// folder channel receives the whole tail (positive control).
@@ -249,6 +250,9 @@ public sealed class EventQueueOverflowAclWireTests
await anonSub.ApplyChangesAsync(ct);
anonA.Status.Error.ShouldBeNull();
// The EqA folder's delivery stalls (#855: nothing of ours defers on its own any more), so the native condition's
// walk suspends there and the rest of the flood is deferred behind it.
using var stall = NotifierStall.Hold(rig.Nm, EqA);
var flood = Task.Factory.StartNew(() =>
{
lock (rig.Nm.Lock)
@@ -276,11 +280,12 @@ public sealed class EventQueueOverflowAclWireTests
throw new ShouldAssertException("revoking a grant under Lock while events were deferred stalled (deadlock)");
}
stall.Release();
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
await Task.Delay(TimeSpan.FromSeconds(2), ct);
await keySub.SetPublishingModeAsync(true, ct);
// Positive control: the still-granted session on the SAME folder channel gets the whole deferred tail.
// Positive control: the still-granted session on the SAME folder gets the whole deferred tail.
var lastNative = $"n{PerCondition - 1}";
await RecordedEvents.WaitUntilAsync(() =>
anonEvents.Has(anonA, lastNative) && anonEvents.Has(anonA, "after-revoke")
@@ -0,0 +1,841 @@
using System.Diagnostics;
using System.Globalization;
using Opc.Ua;
using Opc.Ua.Client;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.Authorization;
using MonitoredItem = Opc.Ua.Client.MonitoredItem;
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
/// <summary>
/// Gitea #855: a <c>ConditionRefresh</c> of an event item on an equipment / raw device FOLDER gets the guarantee the
/// Server object got in #854 — no condition event older than what the refresh replayed reaches the item after
/// RefreshEnd, per-condition order is kept, the per-folder ACL (#826) decides every event and every replayed condition,
/// and neither a reporter nor the refresh ever waits on a stalled delivery.
/// </summary>
/// <remarks>
/// <para>
/// Folder items are now served per item by the node manager (no SDK channel), so in production no notifier defers
/// at all; <see cref="Production_overload_never_defers_and_never_overtakes_a_folder_refresh"/> pins that under a
/// flood with concurrent refreshes. Every other test produces the deferral the cut exists for by stalling delivery
/// at a notifier (<see cref="NotifierStall"/>), the way a full SDK channel used to.
/// </para>
/// <para>
/// The rig's native condition is wired into EqA and EqB and hangs under the raw device folder, so its walk is
/// condition → EqA → … → device folder; a stall at EqA suspends it before EqA's items and every later hop.
/// </para>
/// </remarks>
public sealed class FolderConditionRefreshWireTests
{
private const string EqA = ServerEventRig.EqA;
// Every event after the stalled one is deferred; < 1024 pending, so nothing coalesces.
private const int FloodSize = 500;
/// <param name="enforced">Per-folder ACL on or off (off is the production default).</param>
/// <param name="finalRetained">Whether the condition is still Retain at refresh time (re-sent) or cleared (omitted).</param>
/// <param name="refresh2">ConditionRefresh2 on the folder item instead of ConditionRefresh on its subscription.</param>
/// <param name="modify">
/// The folder item is modified (ModifyMonitoredItems re-subscribes it to its notifier) between the refresh and the
/// drain: its cut must survive, as the Server object's does across an unsubscribe + resubscribe.
/// </param>
[Theory]
[InlineData(false, true, false, false)]
[InlineData(true, true, false, false)]
[InlineData(false, false, false, false)]
[InlineData(true, false, false, false)]
[InlineData(false, true, true, false)]
[InlineData(true, false, true, false)]
[InlineData(false, true, false, true)]
[InlineData(true, false, false, true)]
public async Task Deferred_events_older_than_a_refresh_never_reach_the_refreshed_folder_item_after_RefreshEnd(
bool enforced, bool finalRetained, bool refresh2, bool modify)
{
var ct = TestContext.Current.CancellationToken;
await using var rig = await ServerEventRig.StartAsync("Refresh855", encrypted: false, ct);
rig.KeyRoles = [];
rig.MaterialiseNative();
rig.Enforce(enforced, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
// A second, non-refreshing subscriber on the SAME folder: it must keep the whole deferred tail.
using var otherSession = await rig.ConnectAsync(anonymous: false, ct);
var otherEvents = new RecordedEvents();
var otherSubscription = await RecordedEvents.SubscribeAsync(otherSession, otherEvents, ct);
var otherOnA = RecordedEvents.AddEventItem(otherSubscription, ServerEventRig.UnsNodeId(otherSession, EqA));
await otherSubscription.ApplyChangesAsync(ct);
otherOnA.Status.Error.ShouldBeNull();
using var session = await rig.ConnectAsync(anonymous: false, ct);
var events = new RecordedEvents();
var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(session, EqA));
await subscription.ApplyChangesAsync(ct);
onA.Status.Error.ShouldBeNull();
rig.Project(native: true, Active("seed"));
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "seed") && otherEvents.Has(otherOnA, "seed"));
using (var stall = NotifierStall.Hold(rig.Nm, EqA))
{
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the stall must actually have deferred events");
rig.Project(native: true, finalRetained ? Active("final") : Cleared("final"));
var refresh = Stopwatch.StartNew();
await RefreshAsync(session, subscription, onA, refresh2, ct);
refresh.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(10), "ConditionRefresh must not wait for the stalled delivery");
await RecordedEvents.WaitUntilAsync(() => events.HasType(onA, ObjectTypeIds.RefreshEndEventType));
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the deferred events were still waiting at RefreshEnd");
rig.Nm.ActiveRefreshCuts.ShouldBe(1, "only the refreshed folder item holds a cut");
if (modify)
{
onA.QueueSize = 1001;
await subscription.ApplyChangesAsync(ct);
onA.Status.Error.ShouldBeNull();
}
stall.Release();
}
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
await RecordedEvents.WaitUntilAsync(() => rig.Nm.ActiveRefreshCuts == 0);
rig.Project(native: true, Active("after"));
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "after") && otherEvents.Has(otherOnA, "after"));
await Task.Delay(TimeSpan.FromMilliseconds(300), ct);
var (replayed, afterEnd) = Window(events.Of(onA));
if (finalRetained)
{
replayed.ShouldBe(["final"], "the refresh re-sends the condition's current state");
afterEnd.ShouldBe(["after"], "nothing older than the refreshed state may follow RefreshEnd");
}
else
{
replayed.ShouldBeEmpty("a cleared (Retain=false) condition is not part of a refresh");
afterEnd.ShouldBe(["final", "after"], "stale Retain=true events would re-add the alarm; the Retain=false one agrees");
events.Of(onA).Last(f => RecordedEvents.Message(f) == "final")[RecordedEvents.RetainField].GetBoolean().ShouldBeFalse();
}
rig.Nm.RefreshSupersededDeliveries.ShouldBeGreaterThan(0);
// The other subscriber on the same folder did not refresh: it keeps every deferred event, in order.
var flood = Indices(otherEvents.Of(otherOnA), 'f');
flood.ShouldBe(Enumerable.Range(0, FloodSize).ToList(), "the non-refreshing folder subscriber keeps the whole tail, in order");
otherEvents.Has(otherOnA, "final").ShouldBeTrue();
}
/// <summary>
/// The race the item gate closes, for a folder item: a deferred event is between its supersession check and its
/// queueing to the folder item when the refresh arrives. The refresh must wait for that one delivery, so it lands
/// before the replay rather than after RefreshEnd.
/// </summary>
[Fact]
public async Task A_folder_refresh_waits_for_a_deferred_delivery_already_past_its_check()
{
var ct = TestContext.Current.CancellationToken;
await using var rig = await ServerEventRig.StartAsync("Refresh855Gate", encrypted: false, ct);
rig.KeyRoles = [];
rig.MaterialiseNative();
rig.Enforce(false, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
using var session = await rig.ConnectAsync(anonymous: false, ct);
var events = new RecordedEvents();
var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(session, EqA));
await subscription.ApplyChangesAsync(ct);
rig.Project(native: true, Active("seed"));
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "seed"));
using var entered = new ManualResetEventSlim();
using var proceed = new ManualResetEventSlim();
var probed = 0;
rig.Nm.DeferredFolderDeliveryProbe = () =>
{
if (Interlocked.Increment(ref probed) != 1) return;
entered.Set();
proceed.Wait(TimeSpan.FromSeconds(30));
};
using (NotifierStall.Hold(rig.Nm, EqA))
{
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
rig.Project(native: true, Active("final"));
}
// The released walk parks on its first deferred delivery to the folder item, past the check, inside the gate.
entered.Wait(TimeSpan.FromSeconds(10), ct).ShouldBeTrue("the drain must reach the folder item");
var refresh = session.CallAsync(
ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id);
await Task.Delay(TimeSpan.FromSeconds(1), ct);
events.HasType(onA, ObjectTypeIds.RefreshEndEventType)
.ShouldBeFalse("the refresh must wait for the delivery already inside the item's gate");
proceed.Set();
await refresh.WaitAsync(TimeSpan.FromSeconds(10), ct);
await RecordedEvents.WaitUntilAsync(() => events.HasType(onA, ObjectTypeIds.RefreshEndEventType));
rig.Nm.DeferredFolderDeliveryProbe = null;
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
rig.Project(native: true, Active("after"));
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "after"));
await Task.Delay(TimeSpan.FromMilliseconds(300), ct);
var (window, afterEnd) = Window(events.Of(onA));
window.Count.ShouldBe(2, "the parked delivery, then the replay");
window[0]!.ShouldStartWith("f");
window[1].ShouldBe("final");
afterEnd.ShouldBe(["after"]);
}
/// <summary>
/// The cut path queues its frozen replay OFF <c>Lock</c>. An event newer than the cut that reaches the folder item
/// in that window (a condition delivered synchronously, reported from the probe between the cut and the queueing)
/// must not be followed by its condition's older frozen replay entry.
/// </summary>
[Fact]
public async Task An_event_newer_than_the_folder_cut_is_never_followed_by_its_older_frozen_replay()
{
var ct = TestContext.Current.CancellationToken;
await using var rig = await ServerEventRig.StartAsync("Refresh855Pending", encrypted: false, ct);
rig.KeyRoles = [];
rig.MaterialiseNative();
rig.Enforce(false, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
// A second condition under the device folder only: its walk never reaches the stalled EqA, so it stays synchronous.
const string Free = ServerEventRig.RawDeviceFolder + "/free";
rig.Sink.MaterialiseAlarmCondition(Free, ServerEventRig.RawDeviceFolder, "free", "OffNormalAlarm", 700, AddressSpaceRealm.Raw, isNative: true);
rig.Sink.WriteAlarmCondition(Free, Active("x1"), DateTime.UtcNow, AddressSpaceRealm.Raw);
// EqA needs an event item of its own to be a notifier fan-out the stall can apply to.
using var eqASession = await rig.ConnectAsync(anonymous: false, ct);
var eqAEvents = new RecordedEvents();
var eqASubscription = await RecordedEvents.SubscribeAsync(eqASession, eqAEvents, ct);
RecordedEvents.AddEventItem(eqASubscription, ServerEventRig.UnsNodeId(eqASession, EqA));
await eqASubscription.ApplyChangesAsync(ct);
// The refreshed item is on the raw device folder, which both conditions report through.
using var session = await rig.ConnectAsync(anonymous: false, ct);
var events = new RecordedEvents();
var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
var onDevice = RecordedEvents.AddEventItem(subscription, RawNodeId(session, ServerEventRig.RawDeviceFolder));
await subscription.ApplyChangesAsync(ct);
onDevice.Status.Error.ShouldBeNull();
rig.Project(native: true, Active("seed"));
await RecordedEvents.WaitUntilAsync(() => events.Has(onDevice, "seed"));
var injected = 0;
rig.Nm.DirectRefreshProbe = (cut, _) =>
{
if (cut && Interlocked.Exchange(ref injected, 1) == 0)
rig.Sink.WriteAlarmCondition(Free, Active("x2"), DateTime.UtcNow, AddressSpaceRealm.Raw);
};
using (var stall = NotifierStall.Hold(rig.Nm, EqA))
{
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the cut path runs only while something is deferred");
await session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id);
await RecordedEvents.WaitUntilAsync(() => events.HasType(onDevice, ObjectTypeIds.RefreshEndEventType));
stall.Release();
}
rig.Nm.DirectRefreshProbe = null;
Volatile.Read(ref injected).ShouldBe(1, "the folder refresh took the cut path");
var stream = events.Of(onDevice).ToList();
var end = stream.FindIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshEndEventType);
var free = stream.Take(end).Select(RecordedEvents.Message).Where(m => m is "x1" or "x2").ToList();
free.ShouldContain("x2");
free[^1].ShouldBe("x2", "the older frozen replay of the condition must not follow its newer event");
}
/// <summary>
/// ACL on: a condition the folder refresh omitted because the item may not see it there (its reference into EqA was
/// dropped by the generation) is not withheld afterwards — the reference returns before the drain, and the item must
/// end on the condition's newest state. And while omitted, the replay itself carried nothing of it.
/// </summary>
[Fact]
public async Task Deferred_events_of_a_condition_the_folder_refresh_omitted_by_the_acl_are_not_withheld()
{
var ct = TestContext.Current.CancellationToken;
await using var rig = await ServerEventRig.StartAsync("Refresh855Acl", encrypted: false, ct);
rig.KeyRoles = [];
rig.MaterialiseNative();
NodeAcl[] grants = [KeyEquipmentGrant(EqA)];
var state = rig.Enforce(true, [ServerEventRig.Key()], grants);
Deploy(state, "rev-2", grants, ServerEventRig.EqB); // the native alarm is no longer projected into EqA
using var session = await rig.ConnectAsync(anonymous: false, ct);
var events = new RecordedEvents();
var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(session, EqA));
await subscription.ApplyChangesAsync(ct);
onA.Status.Error.ShouldBeNull();
rig.Project(native: true, Active("seed"));
await Task.Delay(TimeSpan.FromMilliseconds(500), ct);
events.Has(onA, "seed").ShouldBeFalse("the generation does not project the alarm into EqA");
using (var stall = NotifierStall.Hold(rig.Nm, EqA))
{
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
rig.Project(native: true, Active("final"));
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0);
await session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id);
await RecordedEvents.WaitUntilAsync(() => events.HasType(onA, ObjectTypeIds.RefreshEndEventType));
// The reference returns while the condition's events still wait.
Deploy(state, "rev-3", grants, EqA, ServerEventRig.EqB);
stall.Release();
}
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "final"));
var (replayed, after) = Window(events.Of(onA));
replayed.ShouldBeEmpty("the refresh omitted the condition: not visible through EqA at the time");
after.ShouldNotBeEmpty();
after[^1].ShouldBe("final", "the item ends on the newest state of a condition it was never refreshed");
after.Count(m => m!.StartsWith('f')).ShouldBeGreaterThan(0, "its deferred events were delivered, not withheld");
}
/// <summary>
/// ACL on: a grant revoked while a refreshed folder item's deferred events wait. Neither the replay (taken while
/// granted) nor anything after it leaks once revoked; a still-granted subscriber on the same folder gets the tail.
/// </summary>
[Fact]
public async Task A_grant_revoked_after_a_folder_refresh_withholds_every_deferred_event()
{
var ct = TestContext.Current.CancellationToken;
await using var rig = await ServerEventRig.StartAsync("Refresh855Revoke", encrypted: false, ct);
rig.KeyRoles = [];
rig.MaterialiseNative();
var anonGrant = ServerEventRig.EquipmentGrant(EqA, NodePermissions.ReadOnly);
var state = rig.Enforce(true, [ServerEventRig.Key()], KeyEquipmentGrant(EqA), anonGrant);
using var keyed = await rig.ConnectAsync(anonymous: false, ct);
var keyEvents = new RecordedEvents();
var keySub = await RecordedEvents.SubscribeAsync(keyed, keyEvents, ct);
var keyA = RecordedEvents.AddEventItem(keySub, ServerEventRig.UnsNodeId(keyed, EqA));
await keySub.ApplyChangesAsync(ct);
using var anonymous = await rig.ConnectAsync(anonymous: true, ct);
var anonEvents = new RecordedEvents();
var anonSub = await RecordedEvents.SubscribeAsync(anonymous, anonEvents, ct);
var anonA = RecordedEvents.AddEventItem(anonSub, ServerEventRig.UnsNodeId(anonymous, EqA));
await anonSub.ApplyChangesAsync(ct);
rig.Project(native: true, Active("seed"));
await RecordedEvents.WaitUntilAsync(() => keyEvents.Has(keyA, "seed") && anonEvents.Has(anonA, "seed"));
using (var stall = NotifierStall.Hold(rig.Nm, EqA))
{
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
await keyed.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, keySub.Id);
await RecordedEvents.WaitUntilAsync(() => keyEvents.HasType(keyA, ObjectTypeIds.RefreshEndEventType));
state.Apply(
ServerEventRig.Cluster, "rev-2", [ServerEventRig.Key()], [anonGrant], ServerEventRig.ScopePaths(),
ServerEventRig.EquipmentOwnerPaths(), ServerEventRig.ScriptedAlarmOwners(), ServerEventRig.FolderScopes(),
ServerEventRig.TagReferences());
rig.Project(native: true, Active("after-revoke"));
stall.Release();
}
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
await RecordedEvents.WaitUntilAsync(() =>
anonEvents.Has(anonA, $"f{FloodSize - 1}") && anonEvents.Has(anonA, "after-revoke"));
await Task.Delay(TimeSpan.FromSeconds(1), ct);
var (_, after) = Window(keyEvents.Of(keyA));
after.ShouldBeEmpty("after RefreshEnd the revoked session gets nothing: older events are superseded, later ones denied");
keyEvents.Has(keyA, "after-revoke").ShouldBeFalse();
}
/// <summary>
/// #855 review: the cut path decides its replay before taking <c>Lock</c>; a grant revoked between that decision
/// and the queueing of the frozen replay must still withhold the replay (it is re-decided right before each entry).
/// The revocation lands from the refresh probe, which runs after the cut is installed and before the replay is queued.
/// </summary>
/// <param name="serverObject">Refresh a Server-object item instead of a folder item.</param>
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task A_grant_revoked_between_the_cut_and_its_replay_withholds_the_replay(bool serverObject)
{
var ct = TestContext.Current.CancellationToken;
await using var rig = await ServerEventRig.StartAsync("Refresh855RevokeCut", encrypted: false, ct);
rig.KeyRoles = [];
rig.MaterialiseNative();
var anonGrant = ServerEventRig.EquipmentGrant(EqA, NodePermissions.ReadOnly);
var state = rig.Enforce(true, [ServerEventRig.Key()], KeyEquipmentGrant(EqA), anonGrant);
// An anonymous subscriber keeps EqA a notifier fan-out the stall applies to (and stays granted throughout).
using var anonymous = await rig.ConnectAsync(anonymous: true, ct);
var anonEvents = new RecordedEvents();
var anonSub = await RecordedEvents.SubscribeAsync(anonymous, anonEvents, ct);
RecordedEvents.AddEventItem(anonSub, ServerEventRig.UnsNodeId(anonymous, EqA));
await anonSub.ApplyChangesAsync(ct);
using var keyed = await rig.ConnectAsync(anonymous: false, ct);
var keyEvents = new RecordedEvents();
var keySub = await RecordedEvents.SubscribeAsync(keyed, keyEvents, ct);
var item = RecordedEvents.AddEventItem(keySub, serverObject ? ObjectIds.Server : ServerEventRig.UnsNodeId(keyed, EqA));
await keySub.ApplyChangesAsync(ct);
item.Status.Error.ShouldBeNull();
rig.Project(native: true, Active("seed"));
await RecordedEvents.WaitUntilAsync(() => keyEvents.Has(item, "seed"));
var revoked = 0;
void Revoke(bool cut, bool _)
{
if (!cut || Interlocked.Exchange(ref revoked, 1) != 0) return;
state.Apply(
ServerEventRig.Cluster, "rev-2", [ServerEventRig.Key()], [anonGrant], ServerEventRig.ScopePaths(),
ServerEventRig.EquipmentOwnerPaths(), ServerEventRig.ScriptedAlarmOwners(), ServerEventRig.FolderScopes(),
ServerEventRig.TagReferences());
}
if (serverObject) rig.Nm.ServerObjectRefreshProbe = Revoke;
else rig.Nm.DirectRefreshProbe = Revoke;
using (var stall = NotifierStall.Hold(rig.Nm, EqA))
{
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0);
await keyed.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, keySub.Id);
await RecordedEvents.WaitUntilAsync(() => keyEvents.HasType(item, ObjectTypeIds.RefreshEndEventType));
stall.Release();
}
rig.Nm.ServerObjectRefreshProbe = null;
rig.Nm.DirectRefreshProbe = null;
Volatile.Read(ref revoked).ShouldBe(1, "the refresh took the cut path");
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
await Task.Delay(TimeSpan.FromMilliseconds(500), ct);
var (replayed, after) = Window(keyEvents.Of(item));
replayed.ShouldBeEmpty("the grant was revoked before the replay was queued");
after.ShouldBeEmpty("and nothing reaches the item after RefreshEnd either");
}
/// <summary>
/// #855 review 2: a condition removed by a deploy between the cut and the queueing of its frozen replay must not be
/// replayed — its frozen Retain=true state would be a ghost no Retain=false ever follows. ACL off, so nothing but the
/// liveness check stands in the way.
/// </summary>
/// <param name="serverObject">Refresh a Server-object item instead of a folder item.</param>
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task A_condition_removed_between_the_cut_and_its_replay_is_not_replayed(bool serverObject)
{
var ct = TestContext.Current.CancellationToken;
await using var rig = await ServerEventRig.StartAsync("Refresh855RemoveCut", encrypted: false, ct);
rig.KeyRoles = [];
rig.MaterialiseNative();
rig.Enforce(false, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
// A second subscriber keeps EqA a notifier fan-out the stall applies to.
using var other = await rig.ConnectAsync(anonymous: true, ct);
var otherEvents = new RecordedEvents();
var otherSub = await RecordedEvents.SubscribeAsync(other, otherEvents, ct);
RecordedEvents.AddEventItem(otherSub, ServerEventRig.UnsNodeId(other, EqA));
await otherSub.ApplyChangesAsync(ct);
using var session = await rig.ConnectAsync(anonymous: false, ct);
var events = new RecordedEvents();
var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
var item = RecordedEvents.AddEventItem(subscription, serverObject ? ObjectIds.Server : ServerEventRig.UnsNodeId(session, EqA));
await subscription.ApplyChangesAsync(ct);
rig.Project(native: true, Active("seed"));
await RecordedEvents.WaitUntilAsync(() => events.Has(item, "seed"));
var removed = 0;
void Remove(bool cut, bool _)
{
if (cut && Interlocked.Exchange(ref removed, 1) == 0)
rig.Nm.RemoveAlarmConditionNode(ServerEventRig.RawAlarmPath, AddressSpaceRealm.Raw).ShouldBeTrue();
}
if (serverObject) rig.Nm.ServerObjectRefreshProbe = Remove;
else rig.Nm.DirectRefreshProbe = Remove;
using (var stall = NotifierStall.Hold(rig.Nm, EqA))
{
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0);
await session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id);
await RecordedEvents.WaitUntilAsync(() => events.HasType(item, ObjectTypeIds.RefreshEndEventType));
stall.Release();
}
rig.Nm.ServerObjectRefreshProbe = null;
rig.Nm.DirectRefreshProbe = null;
Volatile.Read(ref removed).ShouldBe(1, "the refresh took the cut path");
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
await Task.Delay(TimeSpan.FromMilliseconds(500), ct);
var (replayed, after) = Window(events.Of(item));
replayed.ShouldBeEmpty("the condition was removed before its frozen replay was queued");
after.ShouldBeEmpty("and none of its deferred events follows");
}
/// <summary>
/// #855 review 3: the plain refresh path (nothing deferred — the only path production takes) collects live
/// conditions off <c>Lock</c> and queues them afterwards; a condition removed in between must not be replayed.
/// </summary>
/// <param name="serverObject">Refresh a Server-object item instead of a folder item.</param>
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task A_condition_removed_during_a_plain_refresh_is_not_replayed(bool serverObject)
{
var ct = TestContext.Current.CancellationToken;
await using var rig = await ServerEventRig.StartAsync("Refresh855RemovePlain", encrypted: false, ct);
rig.KeyRoles = [];
rig.MaterialiseNative();
rig.Enforce(false, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
using var session = await rig.ConnectAsync(anonymous: false, ct);
var events = new RecordedEvents();
var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
var item = RecordedEvents.AddEventItem(subscription, serverObject ? ObjectIds.Server : ServerEventRig.UnsNodeId(session, EqA));
await subscription.ApplyChangesAsync(ct);
rig.Project(native: true, Active("seed"));
await RecordedEvents.WaitUntilAsync(() => events.Has(item, "seed"));
var removed = 0;
void Remove(bool cut, bool _)
{
if (!cut && Interlocked.Exchange(ref removed, 1) == 0)
rig.Nm.RemoveAlarmConditionNode(ServerEventRig.RawAlarmPath, AddressSpaceRealm.Raw).ShouldBeTrue();
}
if (serverObject) rig.Nm.ServerObjectRefreshProbe = Remove;
else rig.Nm.DirectRefreshProbe = Remove;
await session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id);
await RecordedEvents.WaitUntilAsync(() => events.HasType(item, ObjectTypeIds.RefreshEndEventType));
rig.Nm.ServerObjectRefreshProbe = null;
rig.Nm.DirectRefreshProbe = null;
Volatile.Read(ref removed).ShouldBe(1, "the plain path ran");
Window(events.Of(item)).Replayed.ShouldBeEmpty("the condition was removed before the replay was queued");
}
/// <summary>
/// #855 review: a delivery that fails inline (here: a hop that throws) is over, not suspended — it must not be booked
/// as a deferral episode (which moved the episode a concurrent plain refresh checks, and cleared cuts under the
/// event-report gate while <c>Lock</c> was held). The Server-object item, served before the failing hop, still gets
/// the event, and the next event is delivered normally.
/// </summary>
[Fact]
public async Task A_delivery_that_throws_is_not_booked_as_a_deferral()
{
var ct = TestContext.Current.CancellationToken;
await using var rig = await ServerEventRig.StartAsync("Refresh855Fault", encrypted: false, ct);
rig.KeyRoles = [];
rig.MaterialiseNative();
rig.Enforce(false, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
using var session = await rig.ConnectAsync(anonymous: false, ct);
var events = new RecordedEvents();
var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(session, EqA));
var onServer = RecordedEvents.AddEventItem(subscription, ObjectIds.Server);
await subscription.ApplyChangesAsync(ct);
var episodes = rig.Nm.DeferralEpisodes;
rig.Nm.NotifierDeliveryStall = node =>
node.NodeId.IdentifierAsString == EqA ? throw new InvalidOperationException("hop fault (test)") : null;
rig.Project(native: true, Active("faulted"));
rig.Nm.NotifierDeliveryStall = null;
rig.Nm.DeferredEventReports.ShouldBe(0);
rig.Nm.DeferralEpisodes.ShouldBe(episodes, "a failed delivery is not a deferral episode");
rig.Project(native: true, Active("next"));
await RecordedEvents.WaitUntilAsync(() =>
events.Has(onServer, "faulted") && events.Has(onServer, "next") && events.Has(onA, "next"));
events.Has(onA, "faulted").ShouldBeFalse("the faulted hop delivered nothing");
}
/// <summary>
/// Production shape, no stall: bursts reported under one <c>Lock</c> hold (an apply pass) and single reports, against
/// folder items that refresh repeatedly while their client is slow to publish. Folder items are served per item, so
/// nothing is ever deferred; each refresh's replay is never older than an event the item already received, nothing
/// after RefreshEnd goes backwards, the stream converges on the newest state, and no replay is queued under <c>Lock</c>.
/// </summary>
/// <param name="enforced">Per-folder ACL on or off.</param>
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Production_overload_never_defers_and_never_overtakes_a_folder_refresh(bool enforced)
{
var ct = TestContext.Current.CancellationToken;
await using var rig = await ServerEventRig.StartAsync("Refresh855Load", encrypted: false, ct);
rig.KeyRoles = [];
rig.MaterialiseNative();
rig.Enforce(enforced, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
using var session = await rig.ConnectAsync(anonymous: false, ct);
var events = new RecordedEvents();
var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(session, EqA));
onA.QueueSize = 100_000; // keep every event so the ordering check sees the whole stream
await subscription.ApplyChangesAsync(ct);
rig.Project(native: true, Active("s0"));
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "s0"));
var probes = new System.Collections.Concurrent.ConcurrentQueue<(bool Cut, bool LockHeld)>();
rig.Nm.DirectRefreshProbe = (cut, held) => probes.Enqueue((cut, held));
using var stop = CancellationTokenSource.CreateLinkedTokenSource(ct);
var counter = 0;
var peakDeferred = 0;
var reporter = Task.Factory.StartNew(() =>
{
while (!stop.IsCancellationRequested)
{
lock (rig.Nm.Lock)
{
for (var i = 0; i < 1000; i++)
{
rig.Project(native: true, Active($"s{Interlocked.Increment(ref counter)}"));
// Sampled inside Lock, after each report: a deferral cannot open and drain unseen (#855 review).
peakDeferred = Math.Max(peakDeferred, rig.Nm.DeferredEventReports);
}
}
rig.Project(native: true, Active($"s{Interlocked.Increment(ref counter)}"));
Thread.Sleep(2);
}
}, stop.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default);
var refusedAsInProgress = 0;
for (var r = 0; r < 30; r++)
{
try
{
await RefreshAsync(session, subscription, onA, refresh2: r % 3 == 2, ct).WaitAsync(TimeSpan.FromSeconds(20), ct);
}
catch (ServiceResultException ex) when (ex.StatusCode == StatusCodes.BadRefreshInProgress)
{
refusedAsInProgress++;
}
await Task.Delay(TimeSpan.FromMilliseconds(80), ct);
}
await stop.CancelAsync();
await reporter.WaitAsync(TimeSpan.FromSeconds(30), CancellationToken.None);
rig.Nm.DirectRefreshProbe = null;
peakDeferred.ShouldBe(0, "no notifier of ours defers: folder items are served per item by the reporter");
rig.Nm.DiscardedEventReports.ShouldBe(0);
var newest = $"s{Volatile.Read(ref counter)}";
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, newest));
await Task.Delay(TimeSpan.FromMilliseconds(500), ct);
CheckStream(events.Of(onA)).ShouldBeEmpty();
refusedAsInProgress.ShouldBeLessThan(30, "most refreshes must actually have run");
probes.ShouldNotBeEmpty();
probes.ShouldAllBe(p => !p.Cut && !p.LockHeld);
RecordedEvents.Message(events.Of(onA).Last(f => f[RecordedEvents.AckedField].TryGetValue(out bool _)))
.ShouldBe(newest, "the stream converges on the newest state");
}
/// <summary>
/// Race / stress with deferral: reports (bursts under <c>Lock</c> and single), repeated refreshes of the folder item
/// and an intermittent stall of its folder's delivery all run at once. Outside a refresh the folder item's stream
/// never goes backwards, each refresh ends on the newest state the item has seen, and it converges. Nothing
/// deadlocks, and neither refresh path queues its replay under <c>Lock</c>.
/// </summary>
/// <param name="enforced">Per-folder ACL on or off.</param>
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Concurrent_reports_folder_refreshes_and_stalls_never_deliver_a_pre_refresh_state_after_RefreshEnd(bool enforced)
{
var ct = TestContext.Current.CancellationToken;
await using var rig = await ServerEventRig.StartAsync("Refresh855Stress", encrypted: false, ct);
rig.KeyRoles = [];
rig.MaterialiseNative();
rig.Enforce(enforced, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
using var session = await rig.ConnectAsync(anonymous: false, ct);
var events = new RecordedEvents();
var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(session, EqA));
onA.QueueSize = 100_000;
await subscription.ApplyChangesAsync(ct);
rig.Project(native: true, Active("s0"));
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "s0"));
var probes = new System.Collections.Concurrent.ConcurrentQueue<(bool Cut, bool LockHeld)>();
rig.Nm.DirectRefreshProbe = (cut, held) => probes.Enqueue((cut, held));
using var stop = CancellationTokenSource.CreateLinkedTokenSource(ct);
var counter = 0;
var reporter = Task.Factory.StartNew(() =>
{
while (!stop.IsCancellationRequested)
{
lock (rig.Nm.Lock)
{
for (var i = 0; i < 300; i++) rig.Project(native: true, Active($"s{Interlocked.Increment(ref counter)}"));
}
rig.Project(native: true, Active($"s{Interlocked.Increment(ref counter)}"));
Thread.Sleep(2);
}
}, stop.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default);
var deferredSeen = 0;
var staller = Task.Factory.StartNew(() =>
{
while (!stop.IsCancellationRequested)
{
using (NotifierStall.Hold(rig.Nm, EqA))
{
Thread.Sleep(300);
if (rig.Nm.DeferredEventReports > 0) Interlocked.Increment(ref deferredSeen);
}
Thread.Sleep(100);
}
}, stop.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default);
var refusedAsInProgress = 0;
var stressDeadline = DateTime.UtcNow + TimeSpan.FromSeconds(90);
for (var r = 0; r < 40 || (Volatile.Read(ref deferredSeen) <= 1 && DateTime.UtcNow < stressDeadline); r++)
{
try
{
await RefreshAsync(session, subscription, onA, refresh2: r % 3 == 2, ct).WaitAsync(TimeSpan.FromSeconds(20), ct);
}
catch (ServiceResultException ex) when (ex.StatusCode == StatusCodes.BadRefreshInProgress)
{
refusedAsInProgress++;
}
await Task.Delay(TimeSpan.FromMilliseconds(80), ct);
}
await stop.CancelAsync();
await reporter.WaitAsync(TimeSpan.FromSeconds(30), CancellationToken.None);
await staller.WaitAsync(TimeSpan.FromSeconds(30), CancellationToken.None);
rig.Nm.DirectRefreshProbe = null;
Volatile.Read(ref deferredSeen).ShouldBeGreaterThan(1, "the stress must actually have run with deferred events");
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
var newest = $"s{Volatile.Read(ref counter)}";
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, newest));
await Task.Delay(TimeSpan.FromMilliseconds(500), ct);
CheckStream(events.Of(onA)).ShouldBeEmpty();
refusedAsInProgress.ShouldBeLessThan(40, "most refreshes must actually have run");
rig.Nm.RefreshSupersededDeliveries.ShouldBeGreaterThan(0, "some folder refresh must actually have overtaken deferred events");
probes.ShouldContain(p => p.Cut, "some folder refresh must have taken the cut path");
probes.ShouldAllBe(p => !p.LockHeld);
RecordedEvents.Message(events.Of(onA).Last(f => f[RecordedEvents.AckedField].TryGetValue(out bool _)))
.ShouldBe(newest, "the stream converges on the newest state");
}
/// <summary>
/// Walks one condition's stream: outside a refresh its states never go backwards, and each refresh ends on the newest
/// state the item has seen (its replay is never older than an event already delivered, and nothing older follows).
/// </summary>
private static List<string> CheckStream(IReadOnlyList<ArrayOf<Variant>> stream)
{
var max = -1;
var last = -1;
var replaying = false;
var violations = new List<string>();
foreach (var f in stream)
{
var type = f[RecordedEvents.EventTypeField].GetNodeId();
if (type == ObjectTypeIds.RefreshStartEventType)
{
replaying = true;
continue;
}
if (type == ObjectTypeIds.RefreshEndEventType)
{
replaying = false;
if (last < max) violations.Add($"a refresh ended on s{last} after s{max} had been delivered");
continue;
}
if (!f[RecordedEvents.AckedField].TryGetValue(out bool _)) continue;
var n = int.Parse(RecordedEvents.Message(f)![1..], CultureInfo.InvariantCulture);
if (!replaying && n < max) violations.Add($"s{n} delivered after s{max}");
max = Math.Max(max, n);
last = n;
}
return violations;
}
private static Task RefreshAsync(ISession session, Subscription subscription, MonitoredItem item, bool refresh2, CancellationToken ct) =>
refresh2
? session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh2, ct, subscription.Id, item.Status.Id)
: session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id);
/// <summary>The condition-event messages inside the item's LAST RefreshStart..RefreshEnd, and those after it.</summary>
private static (List<string?> Replayed, List<string?> AfterEnd) Window(IReadOnlyList<ArrayOf<Variant>> stream)
{
var list = stream.ToList();
var start = list.FindLastIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshStartEventType);
var end = list.FindLastIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshEndEventType);
start.ShouldBeGreaterThanOrEqualTo(0);
end.ShouldBeGreaterThan(start);
static bool IsCondition(ArrayOf<Variant> f) => f[RecordedEvents.AckedField].TryGetValue(out bool _);
return (
list.Skip(start + 1).Take(end - start - 1).Where(IsCondition).Select(RecordedEvents.Message).ToList(),
list.Skip(end + 1).Where(IsCondition).Select(RecordedEvents.Message).ToList());
}
private static List<int> Indices(IReadOnlyList<ArrayOf<Variant>> received, char prefix) =>
received
.Select(RecordedEvents.Message)
.Where(m => m is { Length: > 1 } && m[0] == prefix && char.IsAsciiDigit(m[1]))
.Select(m => int.Parse(m![1..], CultureInfo.InvariantCulture))
.ToList();
private static NodeId RawNodeId(ISession session, string id) =>
new(id, (ushort)session.NamespaceUris.GetIndex(V3NodeIds.RawNamespaceUri));
private static void Deploy(DataPlaneAuthorityState state, string revision, NodeAcl[] grants, params string[] referencing) =>
state.Apply(
ServerEventRig.Cluster, revision, [ServerEventRig.Key()], grants, ServerEventRig.ScopePaths(),
ServerEventRig.EquipmentOwnerPaths(), ServerEventRig.ScriptedAlarmOwners(), ServerEventRig.FolderScopes(),
new Dictionary<string, IReadOnlySet<string>> { [ServerEventRig.RawAlarmPath] = referencing.ToHashSet() });
private static AlarmConditionSnapshot Active(string message) =>
new(true, false, false, true, AlarmShelvingKind.Unshelved, 700, message);
private static AlarmConditionSnapshot Cleared(string message) =>
new(false, true, true, true, AlarmShelvingKind.Unshelved, 700, message);
private static NodeAcl KeyEquipmentGrant(string equipment) => new()
{
NodeAclId = $"acl-key-{equipment}",
ClusterId = ServerEventRig.Cluster,
Principal = ServerEventRig.KeyPrincipal,
ScopeKind = NodeAclScopeKind.Equipment,
ScopeId = $"eq-{equipment}",
PermissionFlags = NodePermissions.ReadOnly,
};
}
@@ -19,6 +19,13 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
/// consumer — a deadlock that also freezes every other operation needing <c>Lock</c>. Publishing is switched off
/// on the client so nothing downstream drains either (the slow / absent consumer case).
/// </summary>
/// <remarks>
/// Since Gitea #855 condition events no longer go through that channel at all: the node manager queues them to a
/// folder's items itself, per item, on the reporter's thread (as it always did for Server-object items), so the same
/// flood now completes without deferring anything. The deadlock test therefore also pins that per-item delivery is in
/// force (it would defer again if the bypass stopped recognising the SDK's fan-out); the ghost test produces its
/// deferral with <see cref="NotifierStall"/>.
/// </remarks>
public sealed class FolderEventFloodWireTests
{
private const string EqA = ServerEventRig.EqA;
@@ -56,8 +63,12 @@ public sealed class FolderEventFloodWireTests
// No publishing: the server-side item queue (QueueSize 1000) is the only sink, as with a stalled client.
await subscription.SetPublishingModeAsync(false, ct);
// #855: the folder's SDK fan-out is the one the node manager recognises and serves per item itself.
rig.Nm.TryGetFolder(EqA)!.OnReportEventAsync?.Target.ShouldBeOfType<Opc.Ua.Server.MonitoredNode2>();
// The whole burst under ONE hold of the node manager's Lock — the shape of any multi-event apply/projection
// pass — starting from an uncached verdict (the item is new), so the delivery task needs Lock for event 1.
var peakDeferred = 0;
var flood = Task.Factory.StartNew(() =>
{
lock (rig.Nm.Lock)
@@ -65,6 +76,7 @@ public sealed class FolderEventFloodWireTests
for (var i = 0; i < FloodSize; i++)
{
rig.Project(native: true, State($"f{i}"));
peakDeferred = Math.Max(peakDeferred, rig.Nm.DeferredEventReports);
Volatile.Write(ref reported, i + 1);
}
}
@@ -83,11 +95,9 @@ public sealed class FolderEventFloodWireTests
+ "on Lock (deadlock)");
}
// The overflow was deferred rather than blocking, and drains once Lock is free.
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
// …and the SDK's channel (up to 4096 accepted events) finishes draining into the item queue, still with
// publishing off, so the count below measures the item queue and not events in transit.
await Task.Delay(TimeSpan.FromSeconds(2), ct);
// #855: nothing was deferred at all — every event was queued to the folder item by the reporter itself.
peakDeferred.ShouldBe(0, "no notifier of ours defers: folder items are served per item, not through the SDK channel");
rig.Nm.DeferredEventReports.ShouldBe(0);
// Delivery is bounded by the item's queue (QueueSize 1000, oldest discarded), not by the flood, and the
// events that survive arrive in report order, ending with the last one reported.
@@ -104,8 +114,7 @@ public sealed class FolderEventFloodWireTests
flooded.Count.ShouldBeLessThanOrEqualTo((int)onA.Status.QueueSize, "delivery is bounded by the item's queue, not the flood");
flooded.ShouldBe(flooded.Order().ToList(), "events leave in the order they were reported");
flooded[^1].ShouldBe(FloodSize - 1);
rig.Nm.DiscardedEventReports.ShouldBeGreaterThan(0,
"past the per-condition budget the oldest waiting events give way to the newest (coalesced)");
rig.Nm.DiscardedEventReports.ShouldBe(0, "nothing was deferred, so nothing was coalesced (the item queue is the bound)");
// And the node manager is live: a later event is decided by the per-folder ACL and delivered.
rig.Project(native: true, State("after"));
@@ -127,7 +136,8 @@ public sealed class FolderEventFloodWireTests
/// #838 review — events deferred behind a full channel must not outlive their condition. A deploy that removes
/// the condition while its events wait used to let them reach Server-object (<c>i=2253</c>) subscribers after the
/// NodeDeleted — a ghost Retain alarm for ScadaBridge — and the folder subscribers too. Every hop of a deferred
/// report now re-checks that the condition is still the live instance.
/// report now re-checks that the condition is still the live instance (#855: also after a stall at a hop, before
/// the folder's items are served).
/// </summary>
/// <param name="enforced">Per-folder / Server-object ACL on or off (off is the production default).</param>
[Theory]
@@ -151,16 +161,20 @@ public sealed class FolderEventFloodWireTests
onServer.Status.Error.ShouldBeNull();
await subscription.SetPublishingModeAsync(false, ct);
// One Lock hold: flood past the channel (the tail is deferred), then the deploy removes the condition.
await Task.Factory.StartNew(() =>
// One Lock hold, delivery stalled at the folder: flood (the tail is deferred), then the deploy removes the condition.
using (var stall = NotifierStall.Hold(rig.Nm, EqA))
{
lock (rig.Nm.Lock)
await Task.Factory.StartNew(() =>
{
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, State($"f{i}"));
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the flood must actually have deferred events");
rig.Nm.RemoveAlarmConditionNode(ServerEventRig.RawAlarmPath, AddressSpaceRealm.Raw).ShouldBeTrue();
}
}, ct, TaskCreationOptions.LongRunning, TaskScheduler.Default).WaitAsync(TimeSpan.FromSeconds(60), ct);
lock (rig.Nm.Lock)
{
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, State($"f{i}"));
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the flood must actually have deferred events");
rig.Nm.RemoveAlarmConditionNode(ServerEventRig.RawAlarmPath, AddressSpaceRealm.Raw).ShouldBeTrue();
}
}, ct, TaskCreationOptions.LongRunning, TaskScheduler.Default).WaitAsync(TimeSpan.FromSeconds(60), ct);
stall.Release();
}
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
rig.Nm.DiscardedEventReports.ShouldBeGreaterThan(0, "the waiting events were dropped, not delivered");
@@ -174,12 +188,15 @@ public sealed class FolderEventFloodWireTests
.Select(m => int.Parse(m![1..], System.Globalization.CultureInfo.InvariantCulture))
.ToArray();
// Positive control: the events reported before the channel filled did arrive (Server object: synchronously at
// the condition's own hop).
// Positive control: the event reported before the stall did arrive at the Server object (synchronously, at the
// condition's own hop, before the walk reached the stalled folder).
Flooded(onServer).ShouldNotBeEmpty();
// None of the deferred tail — which was still waiting when the condition was removed — reached anyone.
Flooded(onServer).Max().ShouldBeLessThan(FloodSize - 1, "no ghost reached the Server-object subscriber");
Flooded(onA).DefaultIfEmpty(-1).Max().ShouldBeLessThan(FloodSize - 1, "no ghost reached the folder subscriber");
// Only the in-flight f0 ever reached the stalled EqA hop, and the condition was gone when the stall released: the
// post-stall liveness re-check must have withheld it (#855 review 4 — "< FloodSize - 1" passed without that check).
Flooded(onA).ShouldBeEmpty("no ghost reached the folder subscriber");
Flooded(onServer).ShouldBe([0], "only the event reported before the stall reached the Server object");
}
private static AlarmConditionSnapshot State(string message) =>
@@ -97,10 +97,10 @@ public sealed class FolderEventVerdictInvalidationTests
rig.Project(native: false, State("s1"));
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "s1"));
// Holding the node-manager lock parks EqA's SDK delivery task: the native event "n1" is queued first and its
// verdict (a cache miss after the first invalidation) needs the lock in the inherited role check — its node is
// never removed, so that check always takes it. The OLD scripted instance's "s2", queued behind it on the same
// single-reader queue, is therefore decided only after the replace and the apply's invalidation.
// Before Gitea #855, holding the node-manager lock parked EqA's SDK delivery task, so the OLD scripted instance's
// "s2" was decided (and its verdict cached) only after the replace and the apply's invalidation. Condition events
// are now decided per item at report time, uncached — "s2" is delivered while its instance is live — and this
// pins that the replace still leaves the new instance's events flowing either way.
lock (rig.Nm.Lock)
{
rig.Nm.CompleteAddressSpaceApply(); // forget the cached allows, so both verdicts must be computed
@@ -0,0 +1,42 @@
using Opc.Ua;
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
/// <summary>
/// Stalls condition-event delivery at one of the node manager's notifiers (Gitea #855): every condition event whose walk
/// reaches that notifier suspends there until <see cref="Release"/>, without blocking the reporter or holding the node
/// manager's <c>Lock</c> — so the condition's later events are deferred, as they were behind a full SDK channel before
/// folder items were delivered per item.
/// </summary>
/// <remarks>
/// The pre-#855 tests produced deferral by holding a folder item's server-side <c>MonitoredItem</c> lock, which parked
/// the SDK's per-node delivery task. Folder items are now queued synchronously by the reporter (as Server-object items
/// always were), so holding an item lock would block the reporter instead; this goes through the node manager's test
/// probe (<see cref="OtOpcUaNodeManager.NotifierDeliveryStall"/>). The probe is consulted only at a notifier that has
/// event items (the hop the per-item delivery serves), so the stalled notifier must carry at least one.
/// </remarks>
internal sealed class NotifierStall : IDisposable
{
private readonly OtOpcUaNodeManager _nm;
private readonly TaskCompletionSource _released = new(TaskCreationOptions.RunContinuationsAsynchronously);
private readonly Func<NodeState, Task?> _probe;
private NotifierStall(OtOpcUaNodeManager nm, string notifierId)
{
_nm = nm;
_probe = node => !_released.Task.IsCompleted && node.NodeId.IdentifierAsString == notifierId ? _released.Task : null;
nm.NotifierDeliveryStall = _probe;
}
/// <summary>Starts stalling delivery at the notifier whose NodeId identifier is <paramref name="notifierId"/>.</summary>
public static NotifierStall Hold(OtOpcUaNodeManager nm, string notifierId) => new(nm, notifierId);
/// <summary>Lets every stalled walk continue (on pool threads, off <c>Lock</c>).</summary>
public void Release() => _released.TrySetResult();
public void Dispose()
{
Release();
if (ReferenceEquals(_nm.NotifierDeliveryStall, _probe)) _nm.NotifierDeliveryStall = null;
}
}