Joseph Doherty
fe48458a41
Review of1622d07f: 1. Perf: every i=2253 refresh ran the whole notifier walk + permission checks + QueueEvent under Lock. Now, with nothing deferred (the common case), the refresh is the SDK-shaped walk off Lock (Lock only to copy the root notifiers), kept only if no deferral episode existed at any moment of it (count 0 before and after, episode unchanged); otherwise the cut path also runs and its later replay supersedes. The cut path takes ACL/role verdicts off Lock, then one short Lock hold: watermark + frozen snapshots of Retain conditions from _alarmConditions (no tree walk) + per-item cut and "replay pending" marker; the replay is queued OFF Lock, one event per hold of the item gate. While pending, a condition event newer than the cut marks its condition so the older frozen entry is dropped rather than following it. A concurrent fast-path refresh cancels a pending cut replay of the same item. Measured, 3000 retained conditions, longest Lock wait seen by another thread during one refresh:1622d07f95-106 ms idle / 102-127 ms deferring; now 0-3 ms idle / 23-32 ms deferring. 2. ACL: a retained condition omitted from the replay for want of a grant is never withheld (RefreshCut.RetainedAtCut); only Retain=true events of a condition omitted because it was NOT retained are. 3. One gate/cut state per item for its lifetime (ConditionalWeakTable keyed by the item): an unsubscribe + resubscribe between refresh and drain keeps the cut. 4. Cuts carry their deferral episode; ClearStaleRefreshCuts runs when the count returns to zero, and a stale-episode cut is inert. 5. Dispose guard for the custom refresh (ObjectDisposedException once disposing; Dispose waits for refreshes in flight). 6. Docs: a refresh waits at most for one in-progress QueueEvent on the item. Tests: resubscribe variants, ACL-grant-lands test, pending-replay probe test, dispose test, 3000-condition Lock-hold measurement with a probe pinning that neither path queues its replay under Lock; stress invariant tightened (never backwards outside a refresh; each refresh ends on the newest delivered state) and tolerant of the SDK's own BadRefreshInProgress overlap refusal. Each new test fails against its targeted mutation.
705 lines
37 KiB
C#
705 lines
37 KiB
C#
using System.Collections;
|
|
using System.Diagnostics;
|
|
using System.Reflection;
|
|
using Opc.Ua;
|
|
using Opc.Ua.Client;
|
|
using Opc.Ua.Server;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Authorization;
|
|
using MonitoredItem = Opc.Ua.Client.MonitoredItem;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
|
|
|
|
/// <summary>
|
|
/// Gitea #854 (from the #838 review): while a folder subscriber's event channel is full, a condition's later events
|
|
/// are deferred in its per-condition queue. A <c>ConditionRefresh</c> on a Server-object (<c>i=2253</c>) item in that
|
|
/// window delivers the condition's CURRENT state between RefreshStart and RefreshEnd — and the deferred events, all
|
|
/// older than that state, used to reach the same item AFTER RefreshEnd, so a client briefly showed (or, for a
|
|
/// condition the refresh had dropped, re-added) a stale alarm. They are now withheld from the refreshed item;
|
|
/// every other subscriber still receives them, and the refresh never waits for the stalled drain.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// The overload is produced deterministically: the folder item's server-side <c>MonitoredItem</c> lock is held, so
|
|
/// the SDK's per-node delivery task parks in <c>QueueEvent</c>, its 4096-slot channel fills, and the condition's
|
|
/// later events are deferred — without holding the node manager's <c>Lock</c>, which the refresh needs.
|
|
/// </remarks>
|
|
public sealed class ConditionRefreshDeferralWireTests
|
|
{
|
|
private const string EqA = ServerEventRig.EqA;
|
|
|
|
// > 4096 (channel) + 1 (parked in the delivery task) + 1 (the in-flight deferred write), < 1024 pending: nothing coalesces.
|
|
private const int FloodSize = 5000;
|
|
|
|
/// <param name="enforced">Per-folder / Server-object ACL on or off (off is the production default).</param>
|
|
/// <param name="finalRetained">
|
|
/// Whether the condition's state at refresh time is still Retain (so the refresh re-sends it) or has cleared
|
|
/// (so the refresh omits it and its final Retain=false event must still arrive).
|
|
/// </param>
|
|
/// <param name="refresh2">ConditionRefresh2 on the single item instead of ConditionRefresh on the subscription.</param>
|
|
/// <param name="resubscribe">
|
|
/// The Server-object item is unsubscribed and resubscribed (the SDK drives this on a monitoring-mode change)
|
|
/// between the refresh and the drain: its cut must survive (#854 review — it lived in the subscription map entry).
|
|
/// </param>
|
|
[Theory]
|
|
[InlineData(false, true, false, false)]
|
|
[InlineData(true, true, false, false)]
|
|
[InlineData(false, false, false, false)]
|
|
[InlineData(true, false, false, false)]
|
|
[InlineData(false, true, true, false)]
|
|
[InlineData(true, false, true, false)]
|
|
[InlineData(false, true, false, true)]
|
|
[InlineData(true, false, false, true)]
|
|
public async Task Deferred_events_older_than_a_refresh_never_reach_the_refreshed_server_object_item_after_RefreshEnd(
|
|
bool enforced, bool finalRetained, bool refresh2, bool resubscribe)
|
|
{
|
|
var ct = TestContext.Current.CancellationToken;
|
|
await using var rig = await ServerEventRig.StartAsync("Refresh854", encrypted: false, ct);
|
|
rig.KeyRoles = [];
|
|
rig.MaterialiseNative();
|
|
rig.Enforce(enforced, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
|
|
|
|
// The folder subscriber whose channel the stall fills, on its own session so its parked item never touches the
|
|
// refreshing client's subscription.
|
|
using var folderSession = await rig.ConnectAsync(anonymous: false, ct);
|
|
var folderEvents = new RecordedEvents();
|
|
var folderSubscription = await RecordedEvents.SubscribeAsync(folderSession, folderEvents, ct);
|
|
var onA = RecordedEvents.AddEventItem(folderSubscription, ServerEventRig.UnsNodeId(folderSession, EqA));
|
|
await folderSubscription.ApplyChangesAsync(ct);
|
|
onA.Status.Error.ShouldBeNull();
|
|
|
|
using var serverSession = await rig.ConnectAsync(anonymous: false, ct);
|
|
var serverEvents = new RecordedEvents();
|
|
var serverSubscription = await RecordedEvents.SubscribeAsync(serverSession, serverEvents, ct);
|
|
var onServer = RecordedEvents.AddEventItem(serverSubscription, ObjectIds.Server);
|
|
await serverSubscription.ApplyChangesAsync(ct);
|
|
onServer.Status.Error.ShouldBeNull();
|
|
|
|
// Warm both paths (the folder item's verdict is cached on its first event).
|
|
rig.Project(native: true, Active("seed"));
|
|
await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, "seed") && serverEvents.Has(onServer, "seed"));
|
|
await folderSubscription.SetPublishingModeAsync(false, ct);
|
|
|
|
using (var stall = ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id)))
|
|
{
|
|
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
|
|
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the flood must actually have deferred events");
|
|
rig.Project(native: true, finalRetained ? Active("final") : Cleared("final"));
|
|
|
|
// The refresh is a service call: it must complete while the drain is still stalled.
|
|
var refresh = Stopwatch.StartNew();
|
|
if (refresh2)
|
|
{
|
|
await serverSession.CallAsync(
|
|
ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh2, ct,
|
|
serverSubscription.Id, onServer.Status.Id);
|
|
}
|
|
else
|
|
{
|
|
await serverSession.CallAsync(
|
|
ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, serverSubscription.Id);
|
|
}
|
|
|
|
refresh.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(10), "ConditionRefresh must not wait for the stalled drain");
|
|
await RecordedEvents.WaitUntilAsync(() => serverEvents.HasType(onServer, ObjectTypeIds.RefreshEndEventType));
|
|
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the deferred events were still waiting at RefreshEnd");
|
|
rig.Nm.ActiveRefreshCuts.ShouldBe(1);
|
|
if (resubscribe)
|
|
{
|
|
var serverItem = FindAllEventsItem(rig.Nm, onServer.Status.Id);
|
|
rig.Nm.SubscribeToAllEvents(null!, serverSubscription.Id, serverItem, unsubscribe: true);
|
|
rig.Nm.SubscribeToAllEvents(null!, serverSubscription.Id, serverItem, unsubscribe: false);
|
|
}
|
|
|
|
stall.Release();
|
|
}
|
|
|
|
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
|
|
// #854 review: a cut does not outlive its deferral episode.
|
|
await RecordedEvents.WaitUntilAsync(() => rig.Nm.ActiveRefreshCuts == 0);
|
|
rig.Nm.ActiveRefreshCuts.ShouldBe(0, "the cut is cleared when the deferral count returns to zero");
|
|
// Positive control: a later transition is delivered (and everything deferred before it has been decided).
|
|
rig.Project(native: true, Active("after"));
|
|
await RecordedEvents.WaitUntilAsync(() => serverEvents.Has(onServer, "after"));
|
|
await Task.Delay(TimeSpan.FromMilliseconds(300), ct);
|
|
|
|
var stream = serverEvents.Of(onServer);
|
|
var start = stream.ToList().FindIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshStartEventType);
|
|
var end = stream.ToList().FindIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshEndEventType);
|
|
start.ShouldBeGreaterThanOrEqualTo(0);
|
|
end.ShouldBeGreaterThan(start);
|
|
var replayed = stream.Skip(start + 1).Take(end - start - 1).Select(RecordedEvents.Message).ToList();
|
|
var afterEnd = stream.Skip(end + 1)
|
|
.Where(f => f[RecordedEvents.AckedField].TryGetValue(out bool _))
|
|
.Select(RecordedEvents.Message)
|
|
.ToList();
|
|
|
|
if (finalRetained)
|
|
{
|
|
replayed.ShouldBe(["final"], "the refresh re-sends the condition's current state");
|
|
afterEnd.ShouldBe(["after"], "nothing older than the refreshed state may follow RefreshEnd");
|
|
}
|
|
else
|
|
{
|
|
replayed.ShouldBeEmpty("a cleared (Retain=false) condition is not part of a refresh");
|
|
// The stale Retain=true events would re-add an alarm the refresh just dropped; the final Retain=false one
|
|
// agrees with the refresh and must still arrive (a client that did not reconcile on RefreshEnd needs it).
|
|
afterEnd.ShouldBe(["final", "after"]);
|
|
stream.Last(f => RecordedEvents.Message(f) == "final")[RecordedEvents.RetainField].GetBoolean().ShouldBeFalse();
|
|
}
|
|
|
|
rig.Nm.RefreshSupersededDeliveries.ShouldBeGreaterThan(0);
|
|
|
|
// The folder subscriber did not refresh: it still receives the deferred events, in order, ending with the newest.
|
|
await folderSubscription.SetPublishingModeAsync(true, ct);
|
|
await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, "after"));
|
|
var folderFlood = folderEvents.Of(onA)
|
|
.Select(RecordedEvents.Message)
|
|
.Where(m => m?.StartsWith('f') == true && m != "final")
|
|
.Select(m => int.Parse(m![1..], System.Globalization.CultureInfo.InvariantCulture))
|
|
.ToList();
|
|
folderFlood.ShouldBe(folderFlood.Order().ToList());
|
|
folderFlood[^1].ShouldBe(FloodSize - 1, "the non-refreshing subscriber keeps the deferred tail");
|
|
folderEvents.Has(onA, "final").ShouldBeTrue();
|
|
}
|
|
|
|
/// <summary>
|
|
/// #854 review (ACL on): a condition the refresh omitted for the item because of its grants — not because it was
|
|
/// cleared — must not have its deferred events withheld: a grant added before the drain lets them through, and the
|
|
/// item must end on the condition's newest state (it would otherwise miss an active alarm entirely).
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Deferred_events_of_a_condition_the_refresh_omitted_for_want_of_a_grant_are_not_withheld()
|
|
{
|
|
var ct = TestContext.Current.CancellationToken;
|
|
await using var rig = await ServerEventRig.StartAsync("Refresh854Acl", encrypted: false, ct);
|
|
rig.KeyRoles = [];
|
|
rig.MaterialiseNative();
|
|
var anonymousOnA = ServerEventRig.EquipmentGrant(EqA, NodePermissions.ReadOnly);
|
|
rig.Enforce(true, [ServerEventRig.Key()], anonymousOnA);
|
|
|
|
// The stalled folder subscriber is anonymous (granted EqA); the refreshing key session holds no grant yet.
|
|
using var folderSession = await rig.ConnectAsync(anonymous: true, ct);
|
|
var folderEvents = new RecordedEvents();
|
|
var folderSubscription = await RecordedEvents.SubscribeAsync(folderSession, folderEvents, ct);
|
|
var onA = RecordedEvents.AddEventItem(folderSubscription, ServerEventRig.UnsNodeId(folderSession, EqA));
|
|
await folderSubscription.ApplyChangesAsync(ct);
|
|
|
|
using var serverSession = await rig.ConnectAsync(anonymous: false, ct);
|
|
var serverEvents = new RecordedEvents();
|
|
var serverSubscription = await RecordedEvents.SubscribeAsync(serverSession, serverEvents, ct);
|
|
var onServer = RecordedEvents.AddEventItem(serverSubscription, ObjectIds.Server);
|
|
await serverSubscription.ApplyChangesAsync(ct);
|
|
|
|
rig.Project(native: true, Active("seed"));
|
|
await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, "seed"));
|
|
serverEvents.Has(onServer, "seed").ShouldBeFalse("the key holds no grant yet");
|
|
await folderSubscription.SetPublishingModeAsync(false, ct);
|
|
|
|
using (var stall = ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id)))
|
|
{
|
|
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
|
|
rig.Project(native: true, Active("final"));
|
|
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0);
|
|
|
|
await serverSession.CallAsync(
|
|
ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, serverSubscription.Id);
|
|
await RecordedEvents.WaitUntilAsync(() => serverEvents.HasType(onServer, ObjectTypeIds.RefreshEndEventType));
|
|
|
|
// The grant arrives while the condition's events still wait.
|
|
rig.Enforce(true, [ServerEventRig.Key()], anonymousOnA, KeyEquipmentGrant(EqA));
|
|
stall.Release();
|
|
}
|
|
|
|
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
|
|
await RecordedEvents.WaitUntilAsync(() => serverEvents.Has(onServer, "final"));
|
|
var stream = serverEvents.Of(onServer).ToList();
|
|
var end = stream.FindIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshEndEventType);
|
|
stream.Take(end).Count(f => f[RecordedEvents.AckedField].TryGetValue(out bool _))
|
|
.ShouldBe(0, "the refresh omitted the condition: no grant");
|
|
var after = stream.Skip(end + 1)
|
|
.Where(f => f[RecordedEvents.AckedField].TryGetValue(out bool _))
|
|
.Select(RecordedEvents.Message)
|
|
.ToList();
|
|
after.ShouldNotBeEmpty();
|
|
after[^1].ShouldBe("final", "the item ends on the newest state of a condition it was never refreshed");
|
|
after.Count(m => m!.StartsWith('f') && m != "final").ShouldBeGreaterThan(0, "its deferred events were delivered, not withheld");
|
|
}
|
|
|
|
/// <summary>
|
|
/// #854 review: the cut path queues its frozen replay OFF <c>Lock</c>. An event newer than the cut that reaches the
|
|
/// item in that window (here a synchronously delivered condition, reported from the probe between the cut and the
|
|
/// queueing) must not be followed by its condition's older frozen replay entry.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task An_event_newer_than_the_cut_is_never_followed_by_its_conditions_older_frozen_replay()
|
|
{
|
|
var ct = TestContext.Current.CancellationToken;
|
|
await using var rig = await ServerEventRig.StartAsync("Refresh854Pending", encrypted: false, ct);
|
|
rig.KeyRoles = [];
|
|
rig.MaterialiseNative();
|
|
rig.Enforce(false, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
|
|
// A second condition under the device folder only: no folder item, so its events are always synchronous.
|
|
const string Free = ServerEventRig.RawDeviceFolder + "/free";
|
|
rig.Sink.MaterialiseAlarmCondition(Free, ServerEventRig.RawDeviceFolder, "free", "OffNormalAlarm", 700, AddressSpaceRealm.Raw, isNative: true);
|
|
rig.Sink.WriteAlarmCondition(Free, Active("x1"), DateTime.UtcNow, AddressSpaceRealm.Raw);
|
|
|
|
using var folderSession = await rig.ConnectAsync(anonymous: false, ct);
|
|
var folderEvents = new RecordedEvents();
|
|
var folderSubscription = await RecordedEvents.SubscribeAsync(folderSession, folderEvents, ct);
|
|
var onA = RecordedEvents.AddEventItem(folderSubscription, ServerEventRig.UnsNodeId(folderSession, EqA));
|
|
await folderSubscription.ApplyChangesAsync(ct);
|
|
|
|
using var serverSession = await rig.ConnectAsync(anonymous: false, ct);
|
|
var serverEvents = new RecordedEvents();
|
|
var serverSubscription = await RecordedEvents.SubscribeAsync(serverSession, serverEvents, ct);
|
|
var onServer = RecordedEvents.AddEventItem(serverSubscription, ObjectIds.Server);
|
|
await serverSubscription.ApplyChangesAsync(ct);
|
|
|
|
rig.Project(native: true, Active("seed"));
|
|
await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, "seed") && serverEvents.Has(onServer, "seed"));
|
|
await folderSubscription.SetPublishingModeAsync(false, ct);
|
|
|
|
var injected = 0;
|
|
rig.Nm.ServerObjectRefreshProbe = (cut, _) =>
|
|
{
|
|
if (cut && Interlocked.Exchange(ref injected, 1) == 0)
|
|
rig.Sink.WriteAlarmCondition(Free, Active("x2"), DateTime.UtcNow, AddressSpaceRealm.Raw);
|
|
};
|
|
|
|
using (var stall = ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id)))
|
|
{
|
|
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
|
|
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the cut path runs only while something is deferred");
|
|
await serverSession.CallAsync(
|
|
ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, serverSubscription.Id);
|
|
await RecordedEvents.WaitUntilAsync(() => serverEvents.HasType(onServer, ObjectTypeIds.RefreshEndEventType));
|
|
stall.Release();
|
|
}
|
|
|
|
rig.Nm.ServerObjectRefreshProbe = null;
|
|
Volatile.Read(ref injected).ShouldBe(1);
|
|
var stream = serverEvents.Of(onServer).ToList();
|
|
var end = stream.FindIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshEndEventType);
|
|
var free = stream.Take(end)
|
|
.Select(RecordedEvents.Message)
|
|
.Where(m => m is "x1" or "x2")
|
|
.ToList();
|
|
free.ShouldContain("x2");
|
|
free[^1].ShouldBe("x2", "the older frozen replay of the condition must not follow its newer event");
|
|
}
|
|
|
|
/// <summary>#854 review: the custom Server-object refresh is refused once the node manager is disposed, like the base's.</summary>
|
|
[Fact]
|
|
public async Task A_refresh_after_disposal_is_refused()
|
|
{
|
|
var ct = TestContext.Current.CancellationToken;
|
|
var rig = await ServerEventRig.StartAsync("Refresh854Dispose", encrypted: false, ct);
|
|
var nm = rig.Nm;
|
|
await rig.DisposeAsync();
|
|
nm.Dispose();
|
|
Should.Throw<ObjectDisposedException>(() => nm.ConditionRefresh(null!, Array.Empty<IEventMonitoredItem>()));
|
|
}
|
|
|
|
/// <summary>
|
|
/// The race the item gate closes, made deterministic: a deferred event is between its supersession check and its
|
|
/// queueing to the Server-object item when the refresh arrives. The refresh must wait for that one delivery (so
|
|
/// the event lands before the replay), not slip in between and let it follow RefreshEnd.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task A_refresh_waits_for_a_deferred_delivery_already_past_its_check_so_it_lands_before_the_replay()
|
|
{
|
|
var ct = TestContext.Current.CancellationToken;
|
|
await using var rig = await ServerEventRig.StartAsync("Refresh854Gate", encrypted: false, ct);
|
|
rig.KeyRoles = [];
|
|
rig.MaterialiseNative();
|
|
rig.Enforce(false, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
|
|
|
|
using var folderSession = await rig.ConnectAsync(anonymous: false, ct);
|
|
var folderEvents = new RecordedEvents();
|
|
var folderSubscription = await RecordedEvents.SubscribeAsync(folderSession, folderEvents, ct);
|
|
var onA = RecordedEvents.AddEventItem(folderSubscription, ServerEventRig.UnsNodeId(folderSession, EqA));
|
|
await folderSubscription.ApplyChangesAsync(ct);
|
|
|
|
using var serverSession = await rig.ConnectAsync(anonymous: false, ct);
|
|
var serverEvents = new RecordedEvents();
|
|
var serverSubscription = await RecordedEvents.SubscribeAsync(serverSession, serverEvents, ct);
|
|
var onServer = RecordedEvents.AddEventItem(serverSubscription, ObjectIds.Server);
|
|
await serverSubscription.ApplyChangesAsync(ct);
|
|
|
|
rig.Project(native: true, Active("seed"));
|
|
await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, "seed") && serverEvents.Has(onServer, "seed"));
|
|
await folderSubscription.SetPublishingModeAsync(false, ct);
|
|
|
|
using var entered = new ManualResetEventSlim();
|
|
using var proceed = new ManualResetEventSlim();
|
|
var probed = 0;
|
|
rig.Nm.DeferredServerObjectDeliveryProbe = () =>
|
|
{
|
|
if (Interlocked.Increment(ref probed) != 1) return;
|
|
entered.Set();
|
|
proceed.Wait(TimeSpan.FromSeconds(30));
|
|
};
|
|
|
|
using (ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id)))
|
|
{
|
|
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
|
|
rig.Project(native: true, Active("final"));
|
|
}
|
|
|
|
// The drain resumes and parks on its first deferred delivery to the Server-object item, past the check.
|
|
entered.Wait(TimeSpan.FromSeconds(10), ct).ShouldBeTrue("the drain must reach the Server-object item");
|
|
var refresh = serverSession.CallAsync(
|
|
ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, serverSubscription.Id);
|
|
await Task.Delay(TimeSpan.FromSeconds(1), ct);
|
|
// (The SDK may answer the Call before the refresh body runs, so the replay is observed on the event stream.)
|
|
serverEvents.HasType(onServer, ObjectTypeIds.RefreshEndEventType)
|
|
.ShouldBeFalse("the refresh must wait for the delivery already inside the item's gate");
|
|
proceed.Set();
|
|
await refresh.WaitAsync(TimeSpan.FromSeconds(10), ct);
|
|
await RecordedEvents.WaitUntilAsync(() => serverEvents.HasType(onServer, ObjectTypeIds.RefreshEndEventType));
|
|
rig.Nm.DeferredServerObjectDeliveryProbe = null;
|
|
|
|
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
|
|
rig.Project(native: true, Active("after"));
|
|
await RecordedEvents.WaitUntilAsync(() => serverEvents.Has(onServer, "after"));
|
|
await Task.Delay(TimeSpan.FromMilliseconds(300), ct);
|
|
|
|
var stream = serverEvents.Of(onServer).ToList();
|
|
var start = stream.FindIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshStartEventType);
|
|
var end = stream.FindIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshEndEventType);
|
|
end.ShouldBeGreaterThan(start);
|
|
var window = stream.Skip(start + 1).Take(end - start - 1).Select(RecordedEvents.Message).ToList();
|
|
window.Count.ShouldBe(2, "the parked delivery, then the replay");
|
|
window[0]!.ShouldStartWith("f");
|
|
window[1].ShouldBe("final");
|
|
stream.Skip(end + 1)
|
|
.Where(f => f[RecordedEvents.AckedField].TryGetValue(out bool _))
|
|
.Select(RecordedEvents.Message)
|
|
.ShouldBe(["after"]);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Race / stress: reports, repeated refreshes and an intermittently stalled drain all run at once. For the
|
|
/// refreshing Server-object item, no condition event delivered after a RefreshEnd may be older than the state that
|
|
/// refresh replayed, and the stream still ends on the newest state. Nothing deadlocks.
|
|
/// </summary>
|
|
/// <param name="enforced">Per-folder / Server-object ACL on or off.</param>
|
|
[Theory]
|
|
[InlineData(false)]
|
|
[InlineData(true)]
|
|
public async Task Concurrent_reports_refreshes_and_drains_never_deliver_a_pre_refresh_state_after_RefreshEnd(bool enforced)
|
|
{
|
|
var ct = TestContext.Current.CancellationToken;
|
|
await using var rig = await ServerEventRig.StartAsync("Refresh854Stress", encrypted: false, ct);
|
|
rig.KeyRoles = [];
|
|
rig.MaterialiseNative();
|
|
rig.Enforce(enforced, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
|
|
|
|
using var folderSession = await rig.ConnectAsync(anonymous: false, ct);
|
|
var folderEvents = new RecordedEvents();
|
|
var folderSubscription = await RecordedEvents.SubscribeAsync(folderSession, folderEvents, ct);
|
|
var onA = RecordedEvents.AddEventItem(folderSubscription, ServerEventRig.UnsNodeId(folderSession, EqA));
|
|
await folderSubscription.ApplyChangesAsync(ct);
|
|
|
|
using var serverSession = await rig.ConnectAsync(anonymous: false, ct);
|
|
var serverEvents = new RecordedEvents();
|
|
var serverSubscription = await RecordedEvents.SubscribeAsync(serverSession, serverEvents, ct);
|
|
var onServer = RecordedEvents.AddEventItem(serverSubscription, ObjectIds.Server);
|
|
onServer.QueueSize = 100_000; // keep every event so the ordering check sees the whole stream
|
|
await serverSubscription.ApplyChangesAsync(ct);
|
|
|
|
rig.Project(native: true, Active("s0"));
|
|
await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, "s0") && serverEvents.Has(onServer, "s0"));
|
|
await folderSubscription.SetPublishingModeAsync(false, ct);
|
|
var item = FindServerItem(rig.Nm, onA.Status.Id);
|
|
|
|
using var stop = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
|
var counter = 0;
|
|
var reporter = Task.Factory.StartNew(() =>
|
|
{
|
|
while (!stop.IsCancellationRequested)
|
|
{
|
|
// Bursts under one Lock hold, like an apply pass, interleaved with single reports.
|
|
lock (rig.Nm.Lock)
|
|
{
|
|
for (var i = 0; i < 500; i++) rig.Project(native: true, Active($"s{Interlocked.Increment(ref counter)}"));
|
|
}
|
|
|
|
rig.Project(native: true, Active($"s{Interlocked.Increment(ref counter)}"));
|
|
Thread.Sleep(2);
|
|
}
|
|
}, stop.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default);
|
|
|
|
var deferredSeen = 0;
|
|
var staller = Task.Factory.StartNew(() =>
|
|
{
|
|
while (!stop.IsCancellationRequested)
|
|
{
|
|
using (ItemLockStall.Hold(item))
|
|
{
|
|
Thread.Sleep(400);
|
|
if (rig.Nm.DeferredEventReports > 0) Interlocked.Increment(ref deferredSeen);
|
|
}
|
|
|
|
Thread.Sleep(100);
|
|
}
|
|
}, stop.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default);
|
|
|
|
var refusedAsInProgress = 0;
|
|
for (var r = 0; r < 40; r++)
|
|
{
|
|
var call = r % 3 == 2
|
|
? serverSession.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh2, ct,
|
|
serverSubscription.Id, onServer.Status.Id)
|
|
: serverSession.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct,
|
|
serverSubscription.Id);
|
|
try
|
|
{
|
|
await call.WaitAsync(TimeSpan.FromSeconds(20), ct);
|
|
}
|
|
catch (ServiceResultException ex) when (ex.StatusCode == StatusCodes.BadRefreshInProgress)
|
|
{
|
|
// The SDK answers the Call before the refresh body runs, so under load the previous refresh of this
|
|
// subscription may still be in progress; the SDK refuses the overlap (its rule, not ours).
|
|
refusedAsInProgress++;
|
|
}
|
|
|
|
await Task.Delay(TimeSpan.FromMilliseconds(80), ct);
|
|
}
|
|
|
|
await stop.CancelAsync();
|
|
await reporter.WaitAsync(TimeSpan.FromSeconds(30), CancellationToken.None);
|
|
await staller.WaitAsync(TimeSpan.FromSeconds(30), CancellationToken.None);
|
|
Volatile.Read(ref deferredSeen).ShouldBeGreaterThan(1, "the stress must actually have run with deferred events");
|
|
|
|
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
|
|
var newest = $"s{Volatile.Read(ref counter)}";
|
|
await folderSubscription.SetPublishingModeAsync(true, ct);
|
|
await RecordedEvents.WaitUntilAsync(() => serverEvents.Has(onServer, newest));
|
|
await Task.Delay(TimeSpan.FromMilliseconds(500), ct);
|
|
|
|
// Walk the Server-object stream (one condition). Outside a refresh its states never go backwards; and each refresh
|
|
// ends on the newest state the item has seen — its replay is never older than an event already delivered, and
|
|
// nothing older than it follows RefreshEnd.
|
|
var max = -1;
|
|
var last = -1;
|
|
var replaying = false;
|
|
var violations = new List<string>();
|
|
foreach (var f in serverEvents.Of(onServer))
|
|
{
|
|
var type = f[RecordedEvents.EventTypeField].GetNodeId();
|
|
if (type == ObjectTypeIds.RefreshStartEventType)
|
|
{
|
|
replaying = true;
|
|
continue;
|
|
}
|
|
|
|
if (type == ObjectTypeIds.RefreshEndEventType)
|
|
{
|
|
replaying = false;
|
|
if (last < max) violations.Add($"a refresh ended on s{last} after s{max} had been delivered");
|
|
continue;
|
|
}
|
|
|
|
if (!f[RecordedEvents.AckedField].TryGetValue(out bool _)) continue;
|
|
var n = int.Parse(RecordedEvents.Message(f)![1..], System.Globalization.CultureInfo.InvariantCulture);
|
|
if (!replaying && n < max) violations.Add($"s{n} delivered after s{max}");
|
|
max = Math.Max(max, n);
|
|
last = n;
|
|
}
|
|
|
|
violations.ShouldBeEmpty();
|
|
refusedAsInProgress.ShouldBeLessThan(40, "most refreshes must actually have run");
|
|
rig.Nm.RefreshSupersededDeliveries.ShouldBeGreaterThan(0, "some refresh must actually have overtaken deferred events");
|
|
RecordedEvents.Message(serverEvents.Of(onServer).Last(f => f[RecordedEvents.AckedField].TryGetValue(out bool _)))
|
|
.ShouldBe(newest, "the stream still converges on the newest state");
|
|
await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, newest));
|
|
}
|
|
|
|
/// <summary>
|
|
/// Review of 1622d07f: a Server-object refresh must not hold the node manager's <c>Lock</c> across a walk of a large
|
|
/// address space — ScadaBridge refreshes on every reconnect, and any session may loop ConditionRefresh. Measures
|
|
/// the longest any other thread waits for <c>Lock</c> while a refresh of 3000 retained conditions runs, with
|
|
/// nothing deferred (the common case) and with a deferral in progress, and pins that the common case does not
|
|
/// build or queue its replay under <c>Lock</c>.
|
|
/// </summary>
|
|
/// <param name="deferring">Whether a deferral episode is in progress when the refresh runs.</param>
|
|
[Theory]
|
|
[InlineData(false)]
|
|
[InlineData(true)]
|
|
public async Task A_refresh_of_a_large_address_space_keeps_the_node_manager_lock_hold_short(bool deferring)
|
|
{
|
|
const int Conditions = 3000;
|
|
var ct = TestContext.Current.CancellationToken;
|
|
await using var rig = await ServerEventRig.StartAsync("Refresh854Lock", encrypted: false, ct);
|
|
rig.KeyRoles = [];
|
|
rig.MaterialiseNative();
|
|
rig.Enforce(false, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
|
|
for (var i = 0; i < Conditions; i++)
|
|
{
|
|
var id = $"{ServerEventRig.RawDeviceFolder}/c{i}";
|
|
rig.Sink.MaterialiseAlarmCondition(id, ServerEventRig.RawDeviceFolder, $"c{i}", "OffNormalAlarm", 700, AddressSpaceRealm.Raw, isNative: true);
|
|
rig.Sink.WriteAlarmCondition(id, Active($"c{i}"), DateTime.UtcNow, AddressSpaceRealm.Raw);
|
|
}
|
|
|
|
using var folderSession = await rig.ConnectAsync(anonymous: false, ct);
|
|
var folderEvents = new RecordedEvents();
|
|
var folderSubscription = await RecordedEvents.SubscribeAsync(folderSession, folderEvents, ct);
|
|
var onA = RecordedEvents.AddEventItem(folderSubscription, ServerEventRig.UnsNodeId(folderSession, EqA));
|
|
await folderSubscription.ApplyChangesAsync(ct);
|
|
|
|
using var serverSession = await rig.ConnectAsync(anonymous: false, ct);
|
|
var serverEvents = new RecordedEvents();
|
|
var serverSubscription = await RecordedEvents.SubscribeAsync(serverSession, serverEvents, ct);
|
|
var onServer = RecordedEvents.AddEventItem(serverSubscription, ObjectIds.Server);
|
|
onServer.QueueSize = 10_000;
|
|
await serverSubscription.ApplyChangesAsync(ct);
|
|
|
|
rig.Project(native: true, Active("seed"));
|
|
await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, "seed") && serverEvents.Has(onServer, "seed"));
|
|
await folderSubscription.SetPublishingModeAsync(false, ct);
|
|
|
|
var stall = deferring ? ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id)) : null;
|
|
try
|
|
{
|
|
if (deferring)
|
|
{
|
|
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
|
|
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0);
|
|
}
|
|
|
|
var probes = new System.Collections.Concurrent.ConcurrentQueue<(bool Cut, bool LockHeld)>();
|
|
rig.Nm.ServerObjectRefreshProbe = (cut, held) => probes.Enqueue((cut, held));
|
|
var waits = new List<double>();
|
|
for (var round = 0; round < 5; round++)
|
|
{
|
|
var ends = serverEvents.Of(onServer).Count(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshEndEventType);
|
|
using var stop = new CancellationTokenSource();
|
|
var maxWait = 0.0;
|
|
var prober = Task.Factory.StartNew(() =>
|
|
{
|
|
var sw = new Stopwatch();
|
|
while (!stop.IsCancellationRequested)
|
|
{
|
|
sw.Restart();
|
|
lock (rig.Nm.Lock) { }
|
|
maxWait = Math.Max(maxWait, sw.Elapsed.TotalMilliseconds);
|
|
Thread.Yield();
|
|
}
|
|
}, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default);
|
|
|
|
await serverSession.CallAsync(
|
|
ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, serverSubscription.Id);
|
|
await RecordedEvents.WaitUntilAsync(() => serverEvents.Of(onServer)
|
|
.Count(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshEndEventType) > ends);
|
|
await stop.CancelAsync();
|
|
await prober;
|
|
waits.Add(maxWait);
|
|
}
|
|
|
|
TestContext.Current.TestOutputHelper?.WriteLine(
|
|
$"#854 lock-hold deferring={deferring} conditions={Conditions}: max Lock wait per refresh (ms) = "
|
|
+ string.Join(", ", waits.Select(w => w.ToString("F1", System.Globalization.CultureInfo.InvariantCulture))));
|
|
// Neither path queues its replay under Lock; only a deferral makes a refresh take the cut path.
|
|
probes.ShouldNotBeEmpty();
|
|
probes.ShouldAllBe(p => !p.LockHeld);
|
|
if (deferring) probes.ShouldContain(p => p.Cut);
|
|
else probes.ShouldAllBe(p => !p.Cut);
|
|
}
|
|
finally
|
|
{
|
|
stall?.Dispose();
|
|
}
|
|
}
|
|
|
|
/// <summary>The server-side Server-object (all-events) item behind a client item.</summary>
|
|
private static IEventMonitoredItem FindAllEventsItem(OtOpcUaNodeManager nm, uint id)
|
|
{
|
|
var map = (IDictionary)typeof(OtOpcUaNodeManager)
|
|
.GetField("_allEventsItems", BindingFlags.Instance | BindingFlags.NonPublic)!
|
|
.GetValue(nm)!;
|
|
foreach (var key in map.Keys)
|
|
{
|
|
if (key is IEventMonitoredItem item && item.Id == id) return item;
|
|
}
|
|
|
|
throw new InvalidOperationException($"no server-side all-events item {id}");
|
|
}
|
|
|
|
/// <summary>The server-side event item behind a client item, from the node manager's direct-item map.</summary>
|
|
private static IEventMonitoredItem FindServerItem(OtOpcUaNodeManager nm, uint id)
|
|
{
|
|
var map = (IDictionary)typeof(OtOpcUaNodeManager)
|
|
.GetField("_eventItems", BindingFlags.Instance | BindingFlags.NonPublic)!
|
|
.GetValue(nm)!;
|
|
foreach (var key in map.Keys)
|
|
{
|
|
if (key is IEventMonitoredItem item && item.Id == id) return item;
|
|
}
|
|
|
|
throw new InvalidOperationException($"no server-side event item {id}");
|
|
}
|
|
|
|
private static AlarmConditionSnapshot Active(string message) =>
|
|
new(true, false, false, true, AlarmShelvingKind.Unshelved, 700, message);
|
|
|
|
private static AlarmConditionSnapshot Cleared(string message) =>
|
|
new(false, true, true, true, AlarmShelvingKind.Unshelved, 700, message);
|
|
|
|
private static NodeAcl KeyEquipmentGrant(string equipment) => new()
|
|
{
|
|
NodeAclId = $"acl-key-{equipment}",
|
|
ClusterId = ServerEventRig.Cluster,
|
|
Principal = ServerEventRig.KeyPrincipal,
|
|
ScopeKind = NodeAclScopeKind.Equipment,
|
|
ScopeId = $"eq-{equipment}",
|
|
PermissionFlags = NodePermissions.ReadOnly,
|
|
};
|
|
|
|
/// <summary>
|
|
/// Holds a server-side monitored item's private lock on a dedicated thread, so the SDK's per-node delivery task
|
|
/// parks in <c>QueueEvent</c> for that item (a stalled subscriber) without the node manager's <c>Lock</c> held.
|
|
/// </summary>
|
|
private sealed class ItemLockStall : IDisposable
|
|
{
|
|
private readonly ManualResetEventSlim _release = new();
|
|
private readonly Thread _thread;
|
|
|
|
private ItemLockStall(IEventMonitoredItem item)
|
|
{
|
|
var gate = (Lock)typeof(Opc.Ua.Server.MonitoredItem)
|
|
.GetField("m_lock", BindingFlags.Instance | BindingFlags.NonPublic)!
|
|
.GetValue(item)!;
|
|
using var held = new ManualResetEventSlim();
|
|
_thread = new Thread(() =>
|
|
{
|
|
gate.Enter();
|
|
try
|
|
{
|
|
held.Set();
|
|
_release.Wait();
|
|
}
|
|
finally { gate.Exit(); }
|
|
}) { IsBackground = true };
|
|
_thread.Start();
|
|
held.Wait();
|
|
}
|
|
|
|
public static ItemLockStall Hold(IEventMonitoredItem item) => new(item);
|
|
|
|
public void Release() => _release.Set();
|
|
|
|
public void Dispose()
|
|
{
|
|
_release.Set();
|
|
_thread.Join();
|
|
_release.Dispose();
|
|
}
|
|
}
|
|
}
|