feat(opcua): per-item delivery for folder event items; folder ConditionRefresh gets the #854 cut (#855)
Condition events reaching one of our notifiers no longer go through the SDK MonitoredNode2 channel: the walk serves the attached items itself, per item and per event (audit guard, uncached INodeManager3 verdict, item gate/cut, QueueEvent), as the Server-object path does. No notifier of ours defers any more; the #838 deferral queue and the #854 cut stay as the safety net and folder items share the cut (RefreshDirectItems, flag on and off). An event whose walk suspends is marked deferred before it suspends. Tests produce deferral with a notifier-stall probe instead of holding a folder item's SDK lock (which would now block the reporter). New FolderConditionRefreshWireTests; design note in docs/plans. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
+15
-55
@@ -24,16 +24,18 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
|
||||
/// every other subscriber still receives them, and the refresh never waits for the stalled drain.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The overload is produced deterministically: the folder item's server-side <c>MonitoredItem</c> lock is held, so
|
||||
/// the SDK's per-node delivery task parks in <c>QueueEvent</c>, its 4096-slot channel fills, and the condition's
|
||||
/// later events are deferred — without holding the node manager's <c>Lock</c>, which the refresh needs.
|
||||
/// The overload is produced deterministically: delivery is stalled at the EqA folder (<see cref="NotifierStall"/>), so the
|
||||
/// condition's walk suspends there and its later events are deferred — without holding the node manager's <c>Lock</c>,
|
||||
/// which the refresh needs. (Before Gitea #855 this held the folder item's SDK lock so the folder's 4096-slot channel
|
||||
/// filled; folder items are now queued per item by the reporter, and no notifier of ours defers on its own.) The
|
||||
/// folder-item twin of these tests is <see cref="FolderConditionRefreshWireTests"/>.
|
||||
/// </remarks>
|
||||
public sealed class ConditionRefreshDeferralWireTests
|
||||
{
|
||||
private const string EqA = ServerEventRig.EqA;
|
||||
|
||||
// > 4096 (channel) + 1 (parked in the delivery task) + 1 (the in-flight deferred write), < 1024 pending: nothing coalesces.
|
||||
private const int FloodSize = 5000;
|
||||
// Every event after the stalled one is deferred; < 1024 pending, so nothing coalesces.
|
||||
private const int FloodSize = 500;
|
||||
|
||||
/// <param name="enforced">Per-folder / Server-object ACL on or off (off is the production default).</param>
|
||||
/// <param name="finalRetained">
|
||||
@@ -84,7 +86,7 @@ public sealed class ConditionRefreshDeferralWireTests
|
||||
await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, "seed") && serverEvents.Has(onServer, "seed"));
|
||||
await folderSubscription.SetPublishingModeAsync(false, ct);
|
||||
|
||||
using (var stall = ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id)))
|
||||
using (var stall = NotifierStall.Hold(rig.Nm, EqA))
|
||||
{
|
||||
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
|
||||
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the flood must actually have deferred events");
|
||||
@@ -200,7 +202,7 @@ public sealed class ConditionRefreshDeferralWireTests
|
||||
serverEvents.Has(onServer, "seed").ShouldBeFalse("the key holds no grant yet");
|
||||
await folderSubscription.SetPublishingModeAsync(false, ct);
|
||||
|
||||
using (var stall = ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id)))
|
||||
using (var stall = NotifierStall.Hold(rig.Nm, EqA))
|
||||
{
|
||||
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
|
||||
rig.Project(native: true, Active("final"));
|
||||
@@ -271,7 +273,7 @@ public sealed class ConditionRefreshDeferralWireTests
|
||||
rig.Sink.WriteAlarmCondition(Free, Active("x2"), DateTime.UtcNow, AddressSpaceRealm.Raw);
|
||||
};
|
||||
|
||||
using (var stall = ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id)))
|
||||
using (var stall = NotifierStall.Hold(rig.Nm, EqA))
|
||||
{
|
||||
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
|
||||
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the cut path runs only while something is deferred");
|
||||
@@ -345,7 +347,7 @@ public sealed class ConditionRefreshDeferralWireTests
|
||||
proceed.Wait(TimeSpan.FromSeconds(30));
|
||||
};
|
||||
|
||||
using (ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id)))
|
||||
using (NotifierStall.Hold(rig.Nm, EqA))
|
||||
{
|
||||
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
|
||||
rig.Project(native: true, Active("final"));
|
||||
@@ -416,7 +418,6 @@ public sealed class ConditionRefreshDeferralWireTests
|
||||
rig.Project(native: true, Active("s0"));
|
||||
await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, "s0") && serverEvents.Has(onServer, "s0"));
|
||||
await folderSubscription.SetPublishingModeAsync(false, ct);
|
||||
var item = FindServerItem(rig.Nm, onA.Status.Id);
|
||||
|
||||
using var stop = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
var counter = 0;
|
||||
@@ -440,7 +441,7 @@ public sealed class ConditionRefreshDeferralWireTests
|
||||
{
|
||||
while (!stop.IsCancellationRequested)
|
||||
{
|
||||
using (ItemLockStall.Hold(item))
|
||||
using (NotifierStall.Hold(rig.Nm, EqA))
|
||||
{
|
||||
Thread.Sleep(400);
|
||||
if (rig.Nm.DeferredEventReports > 0) Interlocked.Increment(ref deferredSeen);
|
||||
@@ -568,7 +569,7 @@ public sealed class ConditionRefreshDeferralWireTests
|
||||
await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, "seed") && serverEvents.Has(onServer, "seed"));
|
||||
await folderSubscription.SetPublishingModeAsync(false, ct);
|
||||
|
||||
var stall = deferring ? ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id)) : null;
|
||||
var stall = deferring ? NotifierStall.Hold(rig.Nm, EqA) : null;
|
||||
try
|
||||
{
|
||||
if (deferring)
|
||||
@@ -622,7 +623,7 @@ public sealed class ConditionRefreshDeferralWireTests
|
||||
}
|
||||
|
||||
/// <summary>The server-side Server-object (all-events) item behind a client item.</summary>
|
||||
private static IEventMonitoredItem FindAllEventsItem(OtOpcUaNodeManager nm, uint id)
|
||||
internal static IEventMonitoredItem FindAllEventsItem(OtOpcUaNodeManager nm, uint id)
|
||||
{
|
||||
var map = (IDictionary)typeof(OtOpcUaNodeManager)
|
||||
.GetField("_allEventsItems", BindingFlags.Instance | BindingFlags.NonPublic)!
|
||||
@@ -636,7 +637,7 @@ public sealed class ConditionRefreshDeferralWireTests
|
||||
}
|
||||
|
||||
/// <summary>The server-side event item behind a client item, from the node manager's direct-item map.</summary>
|
||||
private static IEventMonitoredItem FindServerItem(OtOpcUaNodeManager nm, uint id)
|
||||
internal static IEventMonitoredItem FindServerItem(OtOpcUaNodeManager nm, uint id)
|
||||
{
|
||||
var map = (IDictionary)typeof(OtOpcUaNodeManager)
|
||||
.GetField("_eventItems", BindingFlags.Instance | BindingFlags.NonPublic)!
|
||||
@@ -664,45 +665,4 @@ public sealed class ConditionRefreshDeferralWireTests
|
||||
ScopeId = $"eq-{equipment}",
|
||||
PermissionFlags = NodePermissions.ReadOnly,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Holds a server-side monitored item's private lock on a dedicated thread, so the SDK's per-node delivery task
|
||||
/// parks in <c>QueueEvent</c> for that item (a stalled subscriber) without the node manager's <c>Lock</c> held.
|
||||
/// </summary>
|
||||
private sealed class ItemLockStall : IDisposable
|
||||
{
|
||||
private readonly ManualResetEventSlim _release = new();
|
||||
private readonly Thread _thread;
|
||||
|
||||
private ItemLockStall(IEventMonitoredItem item)
|
||||
{
|
||||
var gate = (Lock)typeof(Opc.Ua.Server.MonitoredItem)
|
||||
.GetField("m_lock", BindingFlags.Instance | BindingFlags.NonPublic)!
|
||||
.GetValue(item)!;
|
||||
using var held = new ManualResetEventSlim();
|
||||
_thread = new Thread(() =>
|
||||
{
|
||||
gate.Enter();
|
||||
try
|
||||
{
|
||||
held.Set();
|
||||
_release.Wait();
|
||||
}
|
||||
finally { gate.Exit(); }
|
||||
}) { IsBackground = true };
|
||||
_thread.Start();
|
||||
held.Wait();
|
||||
}
|
||||
|
||||
public static ItemLockStall Hold(IEventMonitoredItem item) => new(item);
|
||||
|
||||
public void Release() => _release.Set();
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_release.Set();
|
||||
_thread.Join();
|
||||
_release.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+18
-13
@@ -27,13 +27,16 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
|
||||
/// <item>no ACL-denied event reaches an item during the overflow, including deferred events whose grant was
|
||||
/// revoked while they waited.</item>
|
||||
/// </list>
|
||||
/// Since Gitea #855 the folder items are served per item by the reporter (as the Server-object items always were),
|
||||
/// so the flood no longer defers at all — the first test pins that — and the revocation test stalls the folder's
|
||||
/// delivery (<see cref="NotifierStall"/>) to get a deferred backlog.
|
||||
/// </summary>
|
||||
public sealed class EventQueueOverflowAclWireTests
|
||||
{
|
||||
private const string EqA = ServerEventRig.EqA;
|
||||
private const string EqB = ServerEventRig.EqB;
|
||||
|
||||
// Native events per flood: past the SDK's 4096-slot per-notifier channel, so the native condition defers.
|
||||
// Native events per flood: past the SDK's 4096-slot per-notifier channel (which deferred them before #855).
|
||||
private const int PerCondition = 6000;
|
||||
|
||||
/// <param name="singleHold">
|
||||
@@ -150,13 +153,12 @@ public sealed class EventQueueOverflowAclWireTests
|
||||
readLatencies.Max().ShouldBeLessThan(TimeSpan.FromSeconds(5), "a Read must not queue behind the whole flood");
|
||||
(await TimedReadAsync(anonymous, eqA, ct)).ShouldBeLessThan(TimeSpan.FromSeconds(2), "the node manager is free afterwards");
|
||||
|
||||
// Memory: the node manager's own overflow store stayed within its per-condition budget (queued + in flight)…
|
||||
Volatile.Read(ref peakDeferred).ShouldBeLessThanOrEqualTo(
|
||||
2 * (OtOpcUaNodeManager.MaxDeferredEventsPerCondition + 1), "deferral is bounded per condition");
|
||||
if (singleHold) rig.Nm.DiscardedEventReports.ShouldBeGreaterThan(0, "the native tail overflowed and coalesced");
|
||||
// Memory: the node manager held nothing of its own for the overflow (#855: no notifier of ours defers, so the
|
||||
// item queues are the only store, bounded below)…
|
||||
Volatile.Read(ref peakDeferred).ShouldBe(0, "folder items are served per item by the reporter: nothing defers");
|
||||
rig.Nm.DiscardedEventReports.ShouldBe(0);
|
||||
|
||||
// …let the SDK channel finish draining into the (still unpublished) item queues, then publish.
|
||||
await Task.Delay(TimeSpan.FromSeconds(2), ct);
|
||||
// …then publish the (still unpublished) item queues.
|
||||
await keySub.SetPublishingModeAsync(true, ct);
|
||||
var lastNative = $"n{PerCondition - 1}";
|
||||
await RecordedEvents.WaitUntilAsync(() => keyEvents.Has(onB, lastNative) && keyEvents.Has(keyServer, lastNative));
|
||||
@@ -165,10 +167,9 @@ public sealed class EventQueueOverflowAclWireTests
|
||||
foreach (var item in new[] { onB, keyServer })
|
||||
{
|
||||
var received = keyEvents.Of(item);
|
||||
// ACL: not one event of the denied (EqA-owned) scripted alarm, overflow or not. (Under a single hold the
|
||||
// deferred native tail drains last and evicts every synchronously-queued event from the 1000-slot queue, so
|
||||
// there this check cannot see a leak; the "after-s" leg below is the falsifier for that mode — verified by
|
||||
// running with enforcement off: chunked fails here, single-hold fails at "after-s".)
|
||||
// ACL: not one event of the denied (EqA-owned) scripted alarm, overflow or not. (The 1000-slot queue keeps
|
||||
// only the last events of the flood, which interleave n/s, so a leak shows here in both modes; the
|
||||
// "after-s" leg below is a second falsifier.)
|
||||
received.Where(f => RecordedEvents.Message(f)?.StartsWith('s') == true)
|
||||
.ShouldBeEmpty($"{item.DisplayName}: an ACL-denied event leaked during the overflow");
|
||||
|
||||
@@ -211,7 +212,7 @@ public sealed class EventQueueOverflowAclWireTests
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A grant revoked while a deferred backlog waits behind a full notifier channel: the deferred events are decided
|
||||
/// A grant revoked while a deferred backlog waits behind a stalled notifier: the deferred events are decided
|
||||
/// at delivery against the CURRENT generation, so none of them — nor any event reported after the revocation —
|
||||
/// reaches the revoked session's folder or Server-object item, while a still-granted session sharing the same
|
||||
/// folder channel receives the whole tail (positive control).
|
||||
@@ -249,6 +250,9 @@ public sealed class EventQueueOverflowAclWireTests
|
||||
await anonSub.ApplyChangesAsync(ct);
|
||||
anonA.Status.Error.ShouldBeNull();
|
||||
|
||||
// The EqA folder's delivery stalls (#855: nothing of ours defers on its own any more), so the native condition's
|
||||
// walk suspends there and the rest of the flood is deferred behind it.
|
||||
using var stall = NotifierStall.Hold(rig.Nm, EqA);
|
||||
var flood = Task.Factory.StartNew(() =>
|
||||
{
|
||||
lock (rig.Nm.Lock)
|
||||
@@ -276,11 +280,12 @@ public sealed class EventQueueOverflowAclWireTests
|
||||
throw new ShouldAssertException("revoking a grant under Lock while events were deferred stalled (deadlock)");
|
||||
}
|
||||
|
||||
stall.Release();
|
||||
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
|
||||
await Task.Delay(TimeSpan.FromSeconds(2), ct);
|
||||
await keySub.SetPublishingModeAsync(true, ct);
|
||||
|
||||
// Positive control: the still-granted session on the SAME folder channel gets the whole deferred tail.
|
||||
// Positive control: the still-granted session on the SAME folder gets the whole deferred tail.
|
||||
var lastNative = $"n{PerCondition - 1}";
|
||||
await RecordedEvents.WaitUntilAsync(() =>
|
||||
anonEvents.Has(anonA, lastNative) && anonEvents.Has(anonA, "after-revoke")
|
||||
|
||||
+627
@@ -0,0 +1,627 @@
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using Opc.Ua;
|
||||
using Opc.Ua.Client;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Authorization;
|
||||
using MonitoredItem = Opc.Ua.Client.MonitoredItem;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #855: a <c>ConditionRefresh</c> of an event item on an equipment / raw device FOLDER gets the guarantee the
|
||||
/// Server object got in #854 — no condition event older than what the refresh replayed reaches the item after
|
||||
/// RefreshEnd, per-condition order is kept, the per-folder ACL (#826) decides every event and every replayed condition,
|
||||
/// and neither a reporter nor the refresh ever waits on a stalled delivery.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Folder items are now served per item by the node manager (no SDK channel), so in production no notifier defers
|
||||
/// at all; <see cref="Production_overload_never_defers_and_never_overtakes_a_folder_refresh"/> pins that under a
|
||||
/// flood with concurrent refreshes. Every other test produces the deferral the cut exists for by stalling delivery
|
||||
/// at a notifier (<see cref="NotifierStall"/>), the way a full SDK channel used to.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The rig's native condition is wired into EqA and EqB and hangs under the raw device folder, so its walk is
|
||||
/// condition → EqA → … → device folder; a stall at EqA suspends it before EqA's items and every later hop.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class FolderConditionRefreshWireTests
|
||||
{
|
||||
private const string EqA = ServerEventRig.EqA;
|
||||
|
||||
// Every event after the stalled one is deferred; < 1024 pending, so nothing coalesces.
|
||||
private const int FloodSize = 500;
|
||||
|
||||
/// <param name="enforced">Per-folder ACL on or off (off is the production default).</param>
|
||||
/// <param name="finalRetained">Whether the condition is still Retain at refresh time (re-sent) or cleared (omitted).</param>
|
||||
/// <param name="refresh2">ConditionRefresh2 on the folder item instead of ConditionRefresh on its subscription.</param>
|
||||
/// <param name="modify">
|
||||
/// The folder item is modified (ModifyMonitoredItems re-subscribes it to its notifier) between the refresh and the
|
||||
/// drain: its cut must survive, as the Server object's does across an unsubscribe + resubscribe.
|
||||
/// </param>
|
||||
[Theory]
|
||||
[InlineData(false, true, false, false)]
|
||||
[InlineData(true, true, false, false)]
|
||||
[InlineData(false, false, false, false)]
|
||||
[InlineData(true, false, false, false)]
|
||||
[InlineData(false, true, true, false)]
|
||||
[InlineData(true, false, true, false)]
|
||||
[InlineData(false, true, false, true)]
|
||||
[InlineData(true, false, false, true)]
|
||||
public async Task Deferred_events_older_than_a_refresh_never_reach_the_refreshed_folder_item_after_RefreshEnd(
|
||||
bool enforced, bool finalRetained, bool refresh2, bool modify)
|
||||
{
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
await using var rig = await ServerEventRig.StartAsync("Refresh855", encrypted: false, ct);
|
||||
rig.KeyRoles = [];
|
||||
rig.MaterialiseNative();
|
||||
rig.Enforce(enforced, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
|
||||
|
||||
// A second, non-refreshing subscriber on the SAME folder: it must keep the whole deferred tail.
|
||||
using var otherSession = await rig.ConnectAsync(anonymous: false, ct);
|
||||
var otherEvents = new RecordedEvents();
|
||||
var otherSubscription = await RecordedEvents.SubscribeAsync(otherSession, otherEvents, ct);
|
||||
var otherOnA = RecordedEvents.AddEventItem(otherSubscription, ServerEventRig.UnsNodeId(otherSession, EqA));
|
||||
await otherSubscription.ApplyChangesAsync(ct);
|
||||
otherOnA.Status.Error.ShouldBeNull();
|
||||
|
||||
using var session = await rig.ConnectAsync(anonymous: false, ct);
|
||||
var events = new RecordedEvents();
|
||||
var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
|
||||
var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(session, EqA));
|
||||
await subscription.ApplyChangesAsync(ct);
|
||||
onA.Status.Error.ShouldBeNull();
|
||||
|
||||
rig.Project(native: true, Active("seed"));
|
||||
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "seed") && otherEvents.Has(otherOnA, "seed"));
|
||||
|
||||
using (var stall = NotifierStall.Hold(rig.Nm, EqA))
|
||||
{
|
||||
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
|
||||
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the stall must actually have deferred events");
|
||||
rig.Project(native: true, finalRetained ? Active("final") : Cleared("final"));
|
||||
|
||||
var refresh = Stopwatch.StartNew();
|
||||
await RefreshAsync(session, subscription, onA, refresh2, ct);
|
||||
refresh.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(10), "ConditionRefresh must not wait for the stalled delivery");
|
||||
await RecordedEvents.WaitUntilAsync(() => events.HasType(onA, ObjectTypeIds.RefreshEndEventType));
|
||||
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the deferred events were still waiting at RefreshEnd");
|
||||
rig.Nm.ActiveRefreshCuts.ShouldBe(1, "only the refreshed folder item holds a cut");
|
||||
|
||||
if (modify)
|
||||
{
|
||||
onA.QueueSize = 1001;
|
||||
await subscription.ApplyChangesAsync(ct);
|
||||
onA.Status.Error.ShouldBeNull();
|
||||
}
|
||||
|
||||
stall.Release();
|
||||
}
|
||||
|
||||
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
|
||||
await RecordedEvents.WaitUntilAsync(() => rig.Nm.ActiveRefreshCuts == 0);
|
||||
rig.Project(native: true, Active("after"));
|
||||
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "after") && otherEvents.Has(otherOnA, "after"));
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(300), ct);
|
||||
|
||||
var (replayed, afterEnd) = Window(events.Of(onA));
|
||||
if (finalRetained)
|
||||
{
|
||||
replayed.ShouldBe(["final"], "the refresh re-sends the condition's current state");
|
||||
afterEnd.ShouldBe(["after"], "nothing older than the refreshed state may follow RefreshEnd");
|
||||
}
|
||||
else
|
||||
{
|
||||
replayed.ShouldBeEmpty("a cleared (Retain=false) condition is not part of a refresh");
|
||||
afterEnd.ShouldBe(["final", "after"], "stale Retain=true events would re-add the alarm; the Retain=false one agrees");
|
||||
events.Of(onA).Last(f => RecordedEvents.Message(f) == "final")[RecordedEvents.RetainField].GetBoolean().ShouldBeFalse();
|
||||
}
|
||||
|
||||
rig.Nm.RefreshSupersededDeliveries.ShouldBeGreaterThan(0);
|
||||
|
||||
// The other subscriber on the same folder did not refresh: it keeps every deferred event, in order.
|
||||
var flood = Indices(otherEvents.Of(otherOnA), 'f');
|
||||
flood.ShouldBe(Enumerable.Range(0, FloodSize).ToList(), "the non-refreshing folder subscriber keeps the whole tail, in order");
|
||||
otherEvents.Has(otherOnA, "final").ShouldBeTrue();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The race the item gate closes, for a folder item: a deferred event is between its supersession check and its
|
||||
/// queueing to the folder item when the refresh arrives. The refresh must wait for that one delivery, so it lands
|
||||
/// before the replay rather than after RefreshEnd.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_folder_refresh_waits_for_a_deferred_delivery_already_past_its_check()
|
||||
{
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
await using var rig = await ServerEventRig.StartAsync("Refresh855Gate", encrypted: false, ct);
|
||||
rig.KeyRoles = [];
|
||||
rig.MaterialiseNative();
|
||||
rig.Enforce(false, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
|
||||
|
||||
using var session = await rig.ConnectAsync(anonymous: false, ct);
|
||||
var events = new RecordedEvents();
|
||||
var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
|
||||
var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(session, EqA));
|
||||
await subscription.ApplyChangesAsync(ct);
|
||||
|
||||
rig.Project(native: true, Active("seed"));
|
||||
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "seed"));
|
||||
|
||||
using var entered = new ManualResetEventSlim();
|
||||
using var proceed = new ManualResetEventSlim();
|
||||
var probed = 0;
|
||||
rig.Nm.DeferredFolderDeliveryProbe = () =>
|
||||
{
|
||||
if (Interlocked.Increment(ref probed) != 1) return;
|
||||
entered.Set();
|
||||
proceed.Wait(TimeSpan.FromSeconds(30));
|
||||
};
|
||||
|
||||
using (NotifierStall.Hold(rig.Nm, EqA))
|
||||
{
|
||||
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
|
||||
rig.Project(native: true, Active("final"));
|
||||
}
|
||||
|
||||
// The released walk parks on its first deferred delivery to the folder item, past the check, inside the gate.
|
||||
entered.Wait(TimeSpan.FromSeconds(10), ct).ShouldBeTrue("the drain must reach the folder item");
|
||||
var refresh = session.CallAsync(
|
||||
ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id);
|
||||
await Task.Delay(TimeSpan.FromSeconds(1), ct);
|
||||
events.HasType(onA, ObjectTypeIds.RefreshEndEventType)
|
||||
.ShouldBeFalse("the refresh must wait for the delivery already inside the item's gate");
|
||||
proceed.Set();
|
||||
await refresh.WaitAsync(TimeSpan.FromSeconds(10), ct);
|
||||
await RecordedEvents.WaitUntilAsync(() => events.HasType(onA, ObjectTypeIds.RefreshEndEventType));
|
||||
rig.Nm.DeferredFolderDeliveryProbe = null;
|
||||
|
||||
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
|
||||
rig.Project(native: true, Active("after"));
|
||||
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "after"));
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(300), ct);
|
||||
|
||||
var (window, afterEnd) = Window(events.Of(onA));
|
||||
window.Count.ShouldBe(2, "the parked delivery, then the replay");
|
||||
window[0]!.ShouldStartWith("f");
|
||||
window[1].ShouldBe("final");
|
||||
afterEnd.ShouldBe(["after"]);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The cut path queues its frozen replay OFF <c>Lock</c>. An event newer than the cut that reaches the folder item
|
||||
/// in that window (a condition delivered synchronously, reported from the probe between the cut and the queueing)
|
||||
/// must not be followed by its condition's older frozen replay entry.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task An_event_newer_than_the_folder_cut_is_never_followed_by_its_older_frozen_replay()
|
||||
{
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
await using var rig = await ServerEventRig.StartAsync("Refresh855Pending", encrypted: false, ct);
|
||||
rig.KeyRoles = [];
|
||||
rig.MaterialiseNative();
|
||||
rig.Enforce(false, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
|
||||
// A second condition under the device folder only: its walk never reaches the stalled EqA, so it stays synchronous.
|
||||
const string Free = ServerEventRig.RawDeviceFolder + "/free";
|
||||
rig.Sink.MaterialiseAlarmCondition(Free, ServerEventRig.RawDeviceFolder, "free", "OffNormalAlarm", 700, AddressSpaceRealm.Raw, isNative: true);
|
||||
rig.Sink.WriteAlarmCondition(Free, Active("x1"), DateTime.UtcNow, AddressSpaceRealm.Raw);
|
||||
|
||||
// EqA needs an event item of its own to be a notifier fan-out the stall can apply to.
|
||||
using var eqASession = await rig.ConnectAsync(anonymous: false, ct);
|
||||
var eqAEvents = new RecordedEvents();
|
||||
var eqASubscription = await RecordedEvents.SubscribeAsync(eqASession, eqAEvents, ct);
|
||||
RecordedEvents.AddEventItem(eqASubscription, ServerEventRig.UnsNodeId(eqASession, EqA));
|
||||
await eqASubscription.ApplyChangesAsync(ct);
|
||||
|
||||
// The refreshed item is on the raw device folder, which both conditions report through.
|
||||
using var session = await rig.ConnectAsync(anonymous: false, ct);
|
||||
var events = new RecordedEvents();
|
||||
var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
|
||||
var onDevice = RecordedEvents.AddEventItem(subscription, RawNodeId(session, ServerEventRig.RawDeviceFolder));
|
||||
await subscription.ApplyChangesAsync(ct);
|
||||
onDevice.Status.Error.ShouldBeNull();
|
||||
|
||||
rig.Project(native: true, Active("seed"));
|
||||
await RecordedEvents.WaitUntilAsync(() => events.Has(onDevice, "seed"));
|
||||
|
||||
var injected = 0;
|
||||
rig.Nm.DirectRefreshProbe = (cut, _) =>
|
||||
{
|
||||
if (cut && Interlocked.Exchange(ref injected, 1) == 0)
|
||||
rig.Sink.WriteAlarmCondition(Free, Active("x2"), DateTime.UtcNow, AddressSpaceRealm.Raw);
|
||||
};
|
||||
|
||||
using (var stall = NotifierStall.Hold(rig.Nm, EqA))
|
||||
{
|
||||
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
|
||||
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the cut path runs only while something is deferred");
|
||||
await session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id);
|
||||
await RecordedEvents.WaitUntilAsync(() => events.HasType(onDevice, ObjectTypeIds.RefreshEndEventType));
|
||||
stall.Release();
|
||||
}
|
||||
|
||||
rig.Nm.DirectRefreshProbe = null;
|
||||
Volatile.Read(ref injected).ShouldBe(1, "the folder refresh took the cut path");
|
||||
var stream = events.Of(onDevice).ToList();
|
||||
var end = stream.FindIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshEndEventType);
|
||||
var free = stream.Take(end).Select(RecordedEvents.Message).Where(m => m is "x1" or "x2").ToList();
|
||||
free.ShouldContain("x2");
|
||||
free[^1].ShouldBe("x2", "the older frozen replay of the condition must not follow its newer event");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// ACL on: a condition the folder refresh omitted because the item may not see it there (its reference into EqA was
|
||||
/// dropped by the generation) is not withheld afterwards — the reference returns before the drain, and the item must
|
||||
/// end on the condition's newest state. And while omitted, the replay itself carried nothing of it.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Deferred_events_of_a_condition_the_folder_refresh_omitted_by_the_acl_are_not_withheld()
|
||||
{
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
await using var rig = await ServerEventRig.StartAsync("Refresh855Acl", encrypted: false, ct);
|
||||
rig.KeyRoles = [];
|
||||
rig.MaterialiseNative();
|
||||
NodeAcl[] grants = [KeyEquipmentGrant(EqA)];
|
||||
var state = rig.Enforce(true, [ServerEventRig.Key()], grants);
|
||||
Deploy(state, "rev-2", grants, ServerEventRig.EqB); // the native alarm is no longer projected into EqA
|
||||
|
||||
using var session = await rig.ConnectAsync(anonymous: false, ct);
|
||||
var events = new RecordedEvents();
|
||||
var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
|
||||
var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(session, EqA));
|
||||
await subscription.ApplyChangesAsync(ct);
|
||||
onA.Status.Error.ShouldBeNull();
|
||||
|
||||
rig.Project(native: true, Active("seed"));
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(500), ct);
|
||||
events.Has(onA, "seed").ShouldBeFalse("the generation does not project the alarm into EqA");
|
||||
|
||||
using (var stall = NotifierStall.Hold(rig.Nm, EqA))
|
||||
{
|
||||
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
|
||||
rig.Project(native: true, Active("final"));
|
||||
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0);
|
||||
|
||||
await session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id);
|
||||
await RecordedEvents.WaitUntilAsync(() => events.HasType(onA, ObjectTypeIds.RefreshEndEventType));
|
||||
|
||||
// The reference returns while the condition's events still wait.
|
||||
Deploy(state, "rev-3", grants, EqA, ServerEventRig.EqB);
|
||||
stall.Release();
|
||||
}
|
||||
|
||||
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
|
||||
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "final"));
|
||||
var (replayed, after) = Window(events.Of(onA));
|
||||
replayed.ShouldBeEmpty("the refresh omitted the condition: not visible through EqA at the time");
|
||||
after.ShouldNotBeEmpty();
|
||||
after[^1].ShouldBe("final", "the item ends on the newest state of a condition it was never refreshed");
|
||||
after.Count(m => m!.StartsWith('f')).ShouldBeGreaterThan(0, "its deferred events were delivered, not withheld");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// ACL on: a grant revoked while a refreshed folder item's deferred events wait. Neither the replay (taken while
|
||||
/// granted) nor anything after it leaks once revoked; a still-granted subscriber on the same folder gets the tail.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_grant_revoked_after_a_folder_refresh_withholds_every_deferred_event()
|
||||
{
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
await using var rig = await ServerEventRig.StartAsync("Refresh855Revoke", encrypted: false, ct);
|
||||
rig.KeyRoles = [];
|
||||
rig.MaterialiseNative();
|
||||
var anonGrant = ServerEventRig.EquipmentGrant(EqA, NodePermissions.ReadOnly);
|
||||
var state = rig.Enforce(true, [ServerEventRig.Key()], KeyEquipmentGrant(EqA), anonGrant);
|
||||
|
||||
using var keyed = await rig.ConnectAsync(anonymous: false, ct);
|
||||
var keyEvents = new RecordedEvents();
|
||||
var keySub = await RecordedEvents.SubscribeAsync(keyed, keyEvents, ct);
|
||||
var keyA = RecordedEvents.AddEventItem(keySub, ServerEventRig.UnsNodeId(keyed, EqA));
|
||||
await keySub.ApplyChangesAsync(ct);
|
||||
|
||||
using var anonymous = await rig.ConnectAsync(anonymous: true, ct);
|
||||
var anonEvents = new RecordedEvents();
|
||||
var anonSub = await RecordedEvents.SubscribeAsync(anonymous, anonEvents, ct);
|
||||
var anonA = RecordedEvents.AddEventItem(anonSub, ServerEventRig.UnsNodeId(anonymous, EqA));
|
||||
await anonSub.ApplyChangesAsync(ct);
|
||||
|
||||
rig.Project(native: true, Active("seed"));
|
||||
await RecordedEvents.WaitUntilAsync(() => keyEvents.Has(keyA, "seed") && anonEvents.Has(anonA, "seed"));
|
||||
|
||||
using (var stall = NotifierStall.Hold(rig.Nm, EqA))
|
||||
{
|
||||
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
|
||||
await keyed.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, keySub.Id);
|
||||
await RecordedEvents.WaitUntilAsync(() => keyEvents.HasType(keyA, ObjectTypeIds.RefreshEndEventType));
|
||||
|
||||
state.Apply(
|
||||
ServerEventRig.Cluster, "rev-2", [ServerEventRig.Key()], [anonGrant], ServerEventRig.ScopePaths(),
|
||||
ServerEventRig.EquipmentOwnerPaths(), ServerEventRig.ScriptedAlarmOwners(), ServerEventRig.FolderScopes(),
|
||||
ServerEventRig.TagReferences());
|
||||
rig.Project(native: true, Active("after-revoke"));
|
||||
stall.Release();
|
||||
}
|
||||
|
||||
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
|
||||
await RecordedEvents.WaitUntilAsync(() =>
|
||||
anonEvents.Has(anonA, $"f{FloodSize - 1}") && anonEvents.Has(anonA, "after-revoke"));
|
||||
await Task.Delay(TimeSpan.FromSeconds(1), ct);
|
||||
|
||||
var (_, after) = Window(keyEvents.Of(keyA));
|
||||
after.ShouldBeEmpty("after RefreshEnd the revoked session gets nothing: older events are superseded, later ones denied");
|
||||
keyEvents.Has(keyA, "after-revoke").ShouldBeFalse();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Production shape, no stall: bursts reported under one <c>Lock</c> hold (an apply pass) and single reports, against
|
||||
/// folder items that refresh repeatedly while their client is slow to publish. Folder items are served per item, so
|
||||
/// nothing is ever deferred; each refresh's replay is never older than an event the item already received, nothing
|
||||
/// after RefreshEnd goes backwards, the stream converges on the newest state, and no replay is queued under <c>Lock</c>.
|
||||
/// </summary>
|
||||
/// <param name="enforced">Per-folder ACL on or off.</param>
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public async Task Production_overload_never_defers_and_never_overtakes_a_folder_refresh(bool enforced)
|
||||
{
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
await using var rig = await ServerEventRig.StartAsync("Refresh855Load", encrypted: false, ct);
|
||||
rig.KeyRoles = [];
|
||||
rig.MaterialiseNative();
|
||||
rig.Enforce(enforced, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
|
||||
|
||||
using var session = await rig.ConnectAsync(anonymous: false, ct);
|
||||
var events = new RecordedEvents();
|
||||
var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
|
||||
var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(session, EqA));
|
||||
onA.QueueSize = 100_000; // keep every event so the ordering check sees the whole stream
|
||||
await subscription.ApplyChangesAsync(ct);
|
||||
|
||||
rig.Project(native: true, Active("s0"));
|
||||
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "s0"));
|
||||
|
||||
var probes = new System.Collections.Concurrent.ConcurrentQueue<(bool Cut, bool LockHeld)>();
|
||||
rig.Nm.DirectRefreshProbe = (cut, held) => probes.Enqueue((cut, held));
|
||||
|
||||
using var stop = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
var counter = 0;
|
||||
var peakDeferred = 0;
|
||||
var reporter = Task.Factory.StartNew(() =>
|
||||
{
|
||||
while (!stop.IsCancellationRequested)
|
||||
{
|
||||
lock (rig.Nm.Lock)
|
||||
{
|
||||
for (var i = 0; i < 1000; i++) rig.Project(native: true, Active($"s{Interlocked.Increment(ref counter)}"));
|
||||
}
|
||||
|
||||
peakDeferred = Math.Max(peakDeferred, rig.Nm.DeferredEventReports);
|
||||
rig.Project(native: true, Active($"s{Interlocked.Increment(ref counter)}"));
|
||||
Thread.Sleep(2);
|
||||
}
|
||||
}, stop.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default);
|
||||
|
||||
var refusedAsInProgress = 0;
|
||||
for (var r = 0; r < 30; r++)
|
||||
{
|
||||
try
|
||||
{
|
||||
await RefreshAsync(session, subscription, onA, refresh2: r % 3 == 2, ct).WaitAsync(TimeSpan.FromSeconds(20), ct);
|
||||
}
|
||||
catch (ServiceResultException ex) when (ex.StatusCode == StatusCodes.BadRefreshInProgress)
|
||||
{
|
||||
refusedAsInProgress++;
|
||||
}
|
||||
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(80), ct);
|
||||
}
|
||||
|
||||
await stop.CancelAsync();
|
||||
await reporter.WaitAsync(TimeSpan.FromSeconds(30), CancellationToken.None);
|
||||
rig.Nm.DirectRefreshProbe = null;
|
||||
|
||||
peakDeferred.ShouldBe(0, "no notifier of ours defers: folder items are served per item by the reporter");
|
||||
rig.Nm.DiscardedEventReports.ShouldBe(0);
|
||||
var newest = $"s{Volatile.Read(ref counter)}";
|
||||
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, newest));
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(500), ct);
|
||||
|
||||
CheckStream(events.Of(onA)).ShouldBeEmpty();
|
||||
refusedAsInProgress.ShouldBeLessThan(30, "most refreshes must actually have run");
|
||||
probes.ShouldNotBeEmpty();
|
||||
probes.ShouldAllBe(p => !p.Cut && !p.LockHeld);
|
||||
RecordedEvents.Message(events.Of(onA).Last(f => f[RecordedEvents.AckedField].TryGetValue(out bool _)))
|
||||
.ShouldBe(newest, "the stream converges on the newest state");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Race / stress with deferral: reports (bursts under <c>Lock</c> and single), repeated refreshes of the folder item
|
||||
/// and an intermittent stall of its folder's delivery all run at once. Outside a refresh the folder item's stream
|
||||
/// never goes backwards, each refresh ends on the newest state the item has seen, and it converges. Nothing
|
||||
/// deadlocks, and neither refresh path queues its replay under <c>Lock</c>.
|
||||
/// </summary>
|
||||
/// <param name="enforced">Per-folder ACL on or off.</param>
|
||||
[Theory]
|
||||
[InlineData(false)]
|
||||
[InlineData(true)]
|
||||
public async Task Concurrent_reports_folder_refreshes_and_stalls_never_deliver_a_pre_refresh_state_after_RefreshEnd(bool enforced)
|
||||
{
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
await using var rig = await ServerEventRig.StartAsync("Refresh855Stress", encrypted: false, ct);
|
||||
rig.KeyRoles = [];
|
||||
rig.MaterialiseNative();
|
||||
rig.Enforce(enforced, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
|
||||
|
||||
using var session = await rig.ConnectAsync(anonymous: false, ct);
|
||||
var events = new RecordedEvents();
|
||||
var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
|
||||
var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(session, EqA));
|
||||
onA.QueueSize = 100_000;
|
||||
await subscription.ApplyChangesAsync(ct);
|
||||
|
||||
rig.Project(native: true, Active("s0"));
|
||||
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "s0"));
|
||||
|
||||
var probes = new System.Collections.Concurrent.ConcurrentQueue<(bool Cut, bool LockHeld)>();
|
||||
rig.Nm.DirectRefreshProbe = (cut, held) => probes.Enqueue((cut, held));
|
||||
|
||||
using var stop = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
var counter = 0;
|
||||
var reporter = Task.Factory.StartNew(() =>
|
||||
{
|
||||
while (!stop.IsCancellationRequested)
|
||||
{
|
||||
lock (rig.Nm.Lock)
|
||||
{
|
||||
for (var i = 0; i < 300; i++) rig.Project(native: true, Active($"s{Interlocked.Increment(ref counter)}"));
|
||||
}
|
||||
|
||||
rig.Project(native: true, Active($"s{Interlocked.Increment(ref counter)}"));
|
||||
Thread.Sleep(2);
|
||||
}
|
||||
}, stop.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default);
|
||||
|
||||
var deferredSeen = 0;
|
||||
var staller = Task.Factory.StartNew(() =>
|
||||
{
|
||||
while (!stop.IsCancellationRequested)
|
||||
{
|
||||
using (NotifierStall.Hold(rig.Nm, EqA))
|
||||
{
|
||||
Thread.Sleep(300);
|
||||
if (rig.Nm.DeferredEventReports > 0) Interlocked.Increment(ref deferredSeen);
|
||||
}
|
||||
|
||||
Thread.Sleep(100);
|
||||
}
|
||||
}, stop.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default);
|
||||
|
||||
var refusedAsInProgress = 0;
|
||||
var stressDeadline = DateTime.UtcNow + TimeSpan.FromSeconds(90);
|
||||
for (var r = 0; r < 40 || (Volatile.Read(ref deferredSeen) <= 1 && DateTime.UtcNow < stressDeadline); r++)
|
||||
{
|
||||
try
|
||||
{
|
||||
await RefreshAsync(session, subscription, onA, refresh2: r % 3 == 2, ct).WaitAsync(TimeSpan.FromSeconds(20), ct);
|
||||
}
|
||||
catch (ServiceResultException ex) when (ex.StatusCode == StatusCodes.BadRefreshInProgress)
|
||||
{
|
||||
refusedAsInProgress++;
|
||||
}
|
||||
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(80), ct);
|
||||
}
|
||||
|
||||
await stop.CancelAsync();
|
||||
await reporter.WaitAsync(TimeSpan.FromSeconds(30), CancellationToken.None);
|
||||
await staller.WaitAsync(TimeSpan.FromSeconds(30), CancellationToken.None);
|
||||
rig.Nm.DirectRefreshProbe = null;
|
||||
Volatile.Read(ref deferredSeen).ShouldBeGreaterThan(1, "the stress must actually have run with deferred events");
|
||||
|
||||
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
|
||||
var newest = $"s{Volatile.Read(ref counter)}";
|
||||
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, newest));
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(500), ct);
|
||||
|
||||
CheckStream(events.Of(onA)).ShouldBeEmpty();
|
||||
refusedAsInProgress.ShouldBeLessThan(40, "most refreshes must actually have run");
|
||||
rig.Nm.RefreshSupersededDeliveries.ShouldBeGreaterThan(0, "some folder refresh must actually have overtaken deferred events");
|
||||
probes.ShouldContain(p => p.Cut, "some folder refresh must have taken the cut path");
|
||||
probes.ShouldAllBe(p => !p.LockHeld);
|
||||
RecordedEvents.Message(events.Of(onA).Last(f => f[RecordedEvents.AckedField].TryGetValue(out bool _)))
|
||||
.ShouldBe(newest, "the stream converges on the newest state");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Walks one condition's stream: outside a refresh its states never go backwards, and each refresh ends on the newest
|
||||
/// state the item has seen (its replay is never older than an event already delivered, and nothing older follows).
|
||||
/// </summary>
|
||||
private static List<string> CheckStream(IReadOnlyList<ArrayOf<Variant>> stream)
|
||||
{
|
||||
var max = -1;
|
||||
var last = -1;
|
||||
var replaying = false;
|
||||
var violations = new List<string>();
|
||||
foreach (var f in stream)
|
||||
{
|
||||
var type = f[RecordedEvents.EventTypeField].GetNodeId();
|
||||
if (type == ObjectTypeIds.RefreshStartEventType)
|
||||
{
|
||||
replaying = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (type == ObjectTypeIds.RefreshEndEventType)
|
||||
{
|
||||
replaying = false;
|
||||
if (last < max) violations.Add($"a refresh ended on s{last} after s{max} had been delivered");
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!f[RecordedEvents.AckedField].TryGetValue(out bool _)) continue;
|
||||
var n = int.Parse(RecordedEvents.Message(f)![1..], CultureInfo.InvariantCulture);
|
||||
if (!replaying && n < max) violations.Add($"s{n} delivered after s{max}");
|
||||
max = Math.Max(max, n);
|
||||
last = n;
|
||||
}
|
||||
|
||||
return violations;
|
||||
}
|
||||
|
||||
private static Task RefreshAsync(ISession session, Subscription subscription, MonitoredItem item, bool refresh2, CancellationToken ct) =>
|
||||
refresh2
|
||||
? session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh2, ct, subscription.Id, item.Status.Id)
|
||||
: session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id);
|
||||
|
||||
/// <summary>The condition-event messages inside the item's LAST RefreshStart..RefreshEnd, and those after it.</summary>
|
||||
private static (List<string?> Replayed, List<string?> AfterEnd) Window(IReadOnlyList<ArrayOf<Variant>> stream)
|
||||
{
|
||||
var list = stream.ToList();
|
||||
var start = list.FindLastIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshStartEventType);
|
||||
var end = list.FindLastIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshEndEventType);
|
||||
start.ShouldBeGreaterThanOrEqualTo(0);
|
||||
end.ShouldBeGreaterThan(start);
|
||||
static bool IsCondition(ArrayOf<Variant> f) => f[RecordedEvents.AckedField].TryGetValue(out bool _);
|
||||
return (
|
||||
list.Skip(start + 1).Take(end - start - 1).Where(IsCondition).Select(RecordedEvents.Message).ToList(),
|
||||
list.Skip(end + 1).Where(IsCondition).Select(RecordedEvents.Message).ToList());
|
||||
}
|
||||
|
||||
private static List<int> Indices(IReadOnlyList<ArrayOf<Variant>> received, char prefix) =>
|
||||
received
|
||||
.Select(RecordedEvents.Message)
|
||||
.Where(m => m is { Length: > 1 } && m[0] == prefix && char.IsAsciiDigit(m[1]))
|
||||
.Select(m => int.Parse(m![1..], CultureInfo.InvariantCulture))
|
||||
.ToList();
|
||||
|
||||
private static NodeId RawNodeId(ISession session, string id) =>
|
||||
new(id, (ushort)session.NamespaceUris.GetIndex(V3NodeIds.RawNamespaceUri));
|
||||
|
||||
private static void Deploy(DataPlaneAuthorityState state, string revision, NodeAcl[] grants, params string[] referencing) =>
|
||||
state.Apply(
|
||||
ServerEventRig.Cluster, revision, [ServerEventRig.Key()], grants, ServerEventRig.ScopePaths(),
|
||||
ServerEventRig.EquipmentOwnerPaths(), ServerEventRig.ScriptedAlarmOwners(), ServerEventRig.FolderScopes(),
|
||||
new Dictionary<string, IReadOnlySet<string>> { [ServerEventRig.RawAlarmPath] = referencing.ToHashSet() });
|
||||
|
||||
private static AlarmConditionSnapshot Active(string message) =>
|
||||
new(true, false, false, true, AlarmShelvingKind.Unshelved, 700, message);
|
||||
|
||||
private static AlarmConditionSnapshot Cleared(string message) =>
|
||||
new(false, true, true, true, AlarmShelvingKind.Unshelved, 700, message);
|
||||
|
||||
private static NodeAcl KeyEquipmentGrant(string equipment) => new()
|
||||
{
|
||||
NodeAclId = $"acl-key-{equipment}",
|
||||
ClusterId = ServerEventRig.Cluster,
|
||||
Principal = ServerEventRig.KeyPrincipal,
|
||||
ScopeKind = NodeAclScopeKind.Equipment,
|
||||
ScopeId = $"eq-{equipment}",
|
||||
PermissionFlags = NodePermissions.ReadOnly,
|
||||
};
|
||||
}
|
||||
+32
-18
@@ -19,6 +19,13 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
|
||||
/// consumer — a deadlock that also freezes every other operation needing <c>Lock</c>. Publishing is switched off
|
||||
/// on the client so nothing downstream drains either (the slow / absent consumer case).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Since Gitea #855 condition events no longer go through that channel at all: the node manager queues them to a
|
||||
/// folder's items itself, per item, on the reporter's thread (as it always did for Server-object items), so the same
|
||||
/// flood now completes without deferring anything. The deadlock test therefore also pins that per-item delivery is in
|
||||
/// force (it would defer again if the bypass stopped recognising the SDK's fan-out); the ghost test produces its
|
||||
/// deferral with <see cref="NotifierStall"/>.
|
||||
/// </remarks>
|
||||
public sealed class FolderEventFloodWireTests
|
||||
{
|
||||
private const string EqA = ServerEventRig.EqA;
|
||||
@@ -56,8 +63,12 @@ public sealed class FolderEventFloodWireTests
|
||||
// No publishing: the server-side item queue (QueueSize 1000) is the only sink, as with a stalled client.
|
||||
await subscription.SetPublishingModeAsync(false, ct);
|
||||
|
||||
// #855: the folder's SDK fan-out is the one the node manager recognises and serves per item itself.
|
||||
rig.Nm.TryGetFolder(EqA)!.OnReportEventAsync?.Target.ShouldBeOfType<Opc.Ua.Server.MonitoredNode2>();
|
||||
|
||||
// The whole burst under ONE hold of the node manager's Lock — the shape of any multi-event apply/projection
|
||||
// pass — starting from an uncached verdict (the item is new), so the delivery task needs Lock for event 1.
|
||||
var peakDeferred = 0;
|
||||
var flood = Task.Factory.StartNew(() =>
|
||||
{
|
||||
lock (rig.Nm.Lock)
|
||||
@@ -65,6 +76,7 @@ public sealed class FolderEventFloodWireTests
|
||||
for (var i = 0; i < FloodSize; i++)
|
||||
{
|
||||
rig.Project(native: true, State($"f{i}"));
|
||||
peakDeferred = Math.Max(peakDeferred, rig.Nm.DeferredEventReports);
|
||||
Volatile.Write(ref reported, i + 1);
|
||||
}
|
||||
}
|
||||
@@ -83,11 +95,9 @@ public sealed class FolderEventFloodWireTests
|
||||
+ "on Lock (deadlock)");
|
||||
}
|
||||
|
||||
// The overflow was deferred rather than blocking, and drains once Lock is free.
|
||||
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
|
||||
// …and the SDK's channel (up to 4096 accepted events) finishes draining into the item queue, still with
|
||||
// publishing off, so the count below measures the item queue and not events in transit.
|
||||
await Task.Delay(TimeSpan.FromSeconds(2), ct);
|
||||
// #855: nothing was deferred at all — every event was queued to the folder item by the reporter itself.
|
||||
peakDeferred.ShouldBe(0, "no notifier of ours defers: folder items are served per item, not through the SDK channel");
|
||||
rig.Nm.DeferredEventReports.ShouldBe(0);
|
||||
|
||||
// Delivery is bounded by the item's queue (QueueSize 1000, oldest discarded), not by the flood, and the
|
||||
// events that survive arrive in report order, ending with the last one reported.
|
||||
@@ -104,8 +114,7 @@ public sealed class FolderEventFloodWireTests
|
||||
flooded.Count.ShouldBeLessThanOrEqualTo((int)onA.Status.QueueSize, "delivery is bounded by the item's queue, not the flood");
|
||||
flooded.ShouldBe(flooded.Order().ToList(), "events leave in the order they were reported");
|
||||
flooded[^1].ShouldBe(FloodSize - 1);
|
||||
rig.Nm.DiscardedEventReports.ShouldBeGreaterThan(0,
|
||||
"past the per-condition budget the oldest waiting events give way to the newest (coalesced)");
|
||||
rig.Nm.DiscardedEventReports.ShouldBe(0, "nothing was deferred, so nothing was coalesced (the item queue is the bound)");
|
||||
|
||||
// And the node manager is live: a later event is decided by the per-folder ACL and delivered.
|
||||
rig.Project(native: true, State("after"));
|
||||
@@ -127,7 +136,8 @@ public sealed class FolderEventFloodWireTests
|
||||
/// #838 review — events deferred behind a full channel must not outlive their condition. A deploy that removes
|
||||
/// the condition while its events wait used to let them reach Server-object (<c>i=2253</c>) subscribers after the
|
||||
/// NodeDeleted — a ghost Retain alarm for ScadaBridge — and the folder subscribers too. Every hop of a deferred
|
||||
/// report now re-checks that the condition is still the live instance.
|
||||
/// report now re-checks that the condition is still the live instance (#855: also after a stall at a hop, before
|
||||
/// the folder's items are served).
|
||||
/// </summary>
|
||||
/// <param name="enforced">Per-folder / Server-object ACL on or off (off is the production default).</param>
|
||||
[Theory]
|
||||
@@ -151,16 +161,20 @@ public sealed class FolderEventFloodWireTests
|
||||
onServer.Status.Error.ShouldBeNull();
|
||||
await subscription.SetPublishingModeAsync(false, ct);
|
||||
|
||||
// One Lock hold: flood past the channel (the tail is deferred), then the deploy removes the condition.
|
||||
await Task.Factory.StartNew(() =>
|
||||
// One Lock hold, delivery stalled at the folder: flood (the tail is deferred), then the deploy removes the condition.
|
||||
using (var stall = NotifierStall.Hold(rig.Nm, EqA))
|
||||
{
|
||||
lock (rig.Nm.Lock)
|
||||
await Task.Factory.StartNew(() =>
|
||||
{
|
||||
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, State($"f{i}"));
|
||||
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the flood must actually have deferred events");
|
||||
rig.Nm.RemoveAlarmConditionNode(ServerEventRig.RawAlarmPath, AddressSpaceRealm.Raw).ShouldBeTrue();
|
||||
}
|
||||
}, ct, TaskCreationOptions.LongRunning, TaskScheduler.Default).WaitAsync(TimeSpan.FromSeconds(60), ct);
|
||||
lock (rig.Nm.Lock)
|
||||
{
|
||||
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, State($"f{i}"));
|
||||
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the flood must actually have deferred events");
|
||||
rig.Nm.RemoveAlarmConditionNode(ServerEventRig.RawAlarmPath, AddressSpaceRealm.Raw).ShouldBeTrue();
|
||||
}
|
||||
}, ct, TaskCreationOptions.LongRunning, TaskScheduler.Default).WaitAsync(TimeSpan.FromSeconds(60), ct);
|
||||
stall.Release();
|
||||
}
|
||||
|
||||
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
|
||||
rig.Nm.DiscardedEventReports.ShouldBeGreaterThan(0, "the waiting events were dropped, not delivered");
|
||||
@@ -174,8 +188,8 @@ public sealed class FolderEventFloodWireTests
|
||||
.Select(m => int.Parse(m![1..], System.Globalization.CultureInfo.InvariantCulture))
|
||||
.ToArray();
|
||||
|
||||
// Positive control: the events reported before the channel filled did arrive (Server object: synchronously at
|
||||
// the condition's own hop).
|
||||
// Positive control: the event reported before the stall did arrive at the Server object (synchronously, at the
|
||||
// condition's own hop, before the walk reached the stalled folder).
|
||||
Flooded(onServer).ShouldNotBeEmpty();
|
||||
// None of the deferred tail — which was still waiting when the condition was removed — reached anyone.
|
||||
Flooded(onServer).Max().ShouldBeLessThan(FloodSize - 1, "no ghost reached the Server-object subscriber");
|
||||
|
||||
+4
-4
@@ -97,10 +97,10 @@ public sealed class FolderEventVerdictInvalidationTests
|
||||
rig.Project(native: false, State("s1"));
|
||||
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "s1"));
|
||||
|
||||
// Holding the node-manager lock parks EqA's SDK delivery task: the native event "n1" is queued first and its
|
||||
// verdict (a cache miss after the first invalidation) needs the lock in the inherited role check — its node is
|
||||
// never removed, so that check always takes it. The OLD scripted instance's "s2", queued behind it on the same
|
||||
// single-reader queue, is therefore decided only after the replace and the apply's invalidation.
|
||||
// Before Gitea #855, holding the node-manager lock parked EqA's SDK delivery task, so the OLD scripted instance's
|
||||
// "s2" was decided (and its verdict cached) only after the replace and the apply's invalidation. Condition events
|
||||
// are now decided per item at report time, uncached — "s2" is delivered while its instance is live — and this
|
||||
// pins that the replace still leaves the new instance's events flowing either way.
|
||||
lock (rig.Nm.Lock)
|
||||
{
|
||||
rig.Nm.CompleteAddressSpaceApply(); // forget the cached allows, so both verdicts must be computed
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
using Opc.Ua;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// Stalls condition-event delivery at one of the node manager's notifiers (Gitea #855): every condition event whose walk
|
||||
/// reaches that notifier suspends there until <see cref="Release"/>, without blocking the reporter or holding the node
|
||||
/// manager's <c>Lock</c> — so the condition's later events are deferred, as they were behind a full SDK channel before
|
||||
/// folder items were delivered per item.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The pre-#855 tests produced deferral by holding a folder item's server-side <c>MonitoredItem</c> lock, which parked
|
||||
/// the SDK's per-node delivery task. Folder items are now queued synchronously by the reporter (as Server-object items
|
||||
/// always were), so holding an item lock would block the reporter instead; this goes through the node manager's test
|
||||
/// probe (<see cref="OtOpcUaNodeManager.NotifierDeliveryStall"/>). The probe is consulted only at a notifier that has
|
||||
/// event items (the hop the per-item delivery serves), so the stalled notifier must carry at least one.
|
||||
/// </remarks>
|
||||
internal sealed class NotifierStall : IDisposable
|
||||
{
|
||||
private readonly OtOpcUaNodeManager _nm;
|
||||
private readonly TaskCompletionSource _released = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private readonly Func<NodeState, Task?> _probe;
|
||||
|
||||
private NotifierStall(OtOpcUaNodeManager nm, string notifierId)
|
||||
{
|
||||
_nm = nm;
|
||||
_probe = node => !_released.Task.IsCompleted && node.NodeId.IdentifierAsString == notifierId ? _released.Task : null;
|
||||
nm.NotifierDeliveryStall = _probe;
|
||||
}
|
||||
|
||||
/// <summary>Starts stalling delivery at the notifier whose NodeId identifier is <paramref name="notifierId"/>.</summary>
|
||||
public static NotifierStall Hold(OtOpcUaNodeManager nm, string notifierId) => new(nm, notifierId);
|
||||
|
||||
/// <summary>Lets every stalled walk continue (on pool threads, off <c>Lock</c>).</summary>
|
||||
public void Release() => _released.TrySetResult();
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Release();
|
||||
if (ReferenceEquals(_nm.NotifierDeliveryStall, _probe)) _nm.NotifierDeliveryStall = null;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user