fix(opcua): ConditionRefresh on a Server-object item is not overtaken by older deferred condition events (#854)

While a folder subscriber's SDK channel is full, a condition's later events wait in
its #838 deferral queue and reach i=2253 items only when the drain gets to them, off
Lock. A ConditionRefresh in that window replayed the current state, then the older
deferred events followed RefreshEnd (stale state, or a condition the refresh had just
dropped re-added until the newest drained).

- Condition events are reported as ConditionEventSnapshot (an InstanceStateSnapshot
  carrying a report sequence, condition NodeId and Retain); events queued behind a
  deferred report are marked Deferred.
- Server-object refresh (ConditionRefresh and ConditionRefresh2, flag on AND off) now
  runs in RefreshServerObjectItems: watermark, "anything deferred?", the replay and its
  queueing in ONE Lock hold, so the replay is exactly the state at the cut. Per item it
  queues the replay and installs a RefreshCut under the item's gate.
- OnConditionReportEvent holds that gate across "superseded? -> QueueEvent" for
  deferred events only. Withheld from that item alone: events at/below the cut whose
  condition the replay re-sent, or Retain=true events of a condition it omitted; an
  older Retain=false event of an omitted condition is still delivered. Queues, other
  Server-object items and folder items are untouched; refresh never waits for a drain.
- Lock order Lock -> item gate -> SDK item lock, same as a synchronous report.
- Meter otopcua.opcua.condition_events.superseded.
- Folder items keep a documented residual (SDK channel backlog + memoised per-item
  verdict leave no per-event hook).

Tests: ConditionRefreshDeferralWireTests (deterministic overload by holding the folder
item's SDK lock; 6 variants fail before the fix; probe-driven gate test fails with the
gate removed; concurrent reports/refreshes/stall stress test fails before the fix).
This commit is contained in:
Joseph Doherty
2026-11-11 06:37:22 -05:00
parent d1b6cad299
commit 1622d07f2e
5 changed files with 680 additions and 43 deletions
@@ -1386,7 +1386,8 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
// in MaterialiseAlarmCondition), whose fan-outs deliver it to direct notifier subscriptions; the
// condition's own OnReportEvent hands the same snapshot to the Server-object (all-events)
// subscriptions, per item, through the ACL (see OnConditionReportEvent).
var snapshot = new InstanceStateSnapshot();
var snapshot = new ConditionEventSnapshot(
Interlocked.Increment(ref _conditionEventSequence), alarm.NodeId, alarm.Retain?.Value == true);
snapshot.Initialize(SystemContext, alarm);
ReportEventWithoutBlocking(alarm, snapshot);
}
@@ -1434,6 +1435,14 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
/// <summary>Deferred condition events discarded before delivery, any reason (test observation).</summary>
internal long DiscardedEventReports => Interlocked.Read(ref _discardedEventReports);
private long _refreshSupersededDeliveries;
/// <summary>
/// Deferred condition events withheld from a Server-object item because a ConditionRefresh on that item already
/// delivered a newer state (Gitea #854) — one per item and event (test observation).
/// </summary>
internal long RefreshSupersededDeliveries => Interlocked.Read(ref _refreshSupersededDeliveries);
/// <summary>
/// Reports a condition event through the condition's notifier chain WITHOUT blocking the caller, in the order
/// this condition's events were reported, and never for a condition that is no longer live (Gitea #838).
@@ -1517,6 +1526,8 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
/// <summary>Queues a later event behind a condition's deferred one, coalescing past the budget. Under the gate.</summary>
private void EnqueueDeferred(AlarmConditionState condition, DeferredConditionReports queue, IFilterTarget e)
{
// Delivered off Lock from here on, so a ConditionRefresh can overtake it (#854, see OnConditionReportEvent).
if (e is ConditionEventSnapshot snapshot) snapshot.Deferred = true;
queue.Pending.Enqueue(e);
CountDeferred(+1, condition);
while (queue.Pending.Count > 1
@@ -1651,6 +1662,65 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
public readonly Queue<IFilterTarget> Pending = new();
}
/// <summary>
/// The report order of condition events (#854): incremented under <c>Lock</c> by <see cref="ReportConditionEvent"/>,
/// read under <c>Lock</c> by a Server-object ConditionRefresh as its cut.
/// </summary>
private long _conditionEventSequence;
/// <summary>
/// A condition event as <see cref="ReportConditionEvent"/> reports it: the SDK's frozen snapshot, plus what a
/// Server-object ConditionRefresh needs to recognise it as superseded (Gitea #854). Still an
/// <see cref="InstanceStateSnapshot"/>, so the SDK's event identity, filtered-retain and queue handling treat it
/// exactly as before.
/// </summary>
/// <param name="sequence">Its place in report order (<see cref="_conditionEventSequence"/>).</param>
/// <param name="conditionId">The reporting condition's NodeId (not the instance: a cut must not pin a removed node).</param>
/// <param name="retain">The condition's <c>Retain</c> as reported.</param>
private sealed class ConditionEventSnapshot(long sequence, NodeId conditionId, bool retain) : InstanceStateSnapshot
{
public long Sequence { get; } = sequence;
public NodeId ConditionId { get; } = conditionId;
public bool Retain { get; } = retain;
/// <summary>
/// Set (under the event-report gate, before the drain can dequeue it) when the event waits in a deferral queue:
/// only such an event reaches Server-object items off <c>Lock</c>, which is the only way a refresh can be overtaken.
/// </summary>
public bool Deferred { get; set; }
}
/// <summary>
/// Per Server-object item state (#854): the gate that orders a deferred delivery to the item against a refresh of
/// the item, and the cut of the item's latest refresh while deferred events may still be older than it.
/// </summary>
private sealed class AllEventsItemState
{
/// <summary>Held across "is this deferred event superseded? → QueueEvent", and across a refresh's
/// "QueueEvent the replay → install the cut". Only ever taken innermost (nothing but the item's own lock under it).</summary>
public readonly object Gate = new();
/// <summary>Under <see cref="Gate"/>.</summary>
public RefreshCut? Cut;
}
/// <summary>What a Server-object item's latest ConditionRefresh conveyed (#854).</summary>
/// <param name="Watermark">Every condition event with a sequence at or below this was reported before the replay was read.</param>
/// <param name="Refreshed">The conditions the replay re-sent to the item; the others were Retain=false (or not visible).</param>
private sealed record RefreshCut(long Watermark, HashSet<NodeId> Refreshed)
{
/// <summary>
/// Whether the refresh already told the item something at least as new as <paramref name="e"/>: a re-sent
/// condition's current state supersedes every older event of it; for a condition the replay OMITTED (it was not
/// retained), an older Retain=true event would wrongly re-add it, while an older Retain=false event agrees with
/// the omission and is still delivered (a client that did not reconcile on RefreshEnd needs it).
/// </summary>
public bool Supersedes(ConditionEventSnapshot e) =>
e.Sequence <= Watermark && (e.Retain || Refreshed.Contains(e.ConditionId));
}
/// <summary>
/// The gate-relevant slice of a Part 9 condition's state — exactly the fields that drive a
@@ -5292,7 +5362,7 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
/// unsubscribe arms, which the SDK's <c>MasterNodeManager</c> drives for every create, modify,
/// transfer, restore and delete of an all-events item, subscription teardown included.
/// </remarks>
private readonly ConcurrentDictionary<IEventMonitoredItem, byte> _allEventsItems = new();
private readonly ConcurrentDictionary<IEventMonitoredItem, AllEventsItemState> _allEventsItems = new();
// The SDK's base implementation attaches a Server-object item to the event
// fan-out of every one of this manager's root notifiers, so every alarm in the server reached it
@@ -5314,7 +5384,7 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
lock (Lock)
{
if (unsubscribe) _allEventsItems.TryRemove(monitoredItem, out _);
else _allEventsItems[monitoredItem] = 0;
else _allEventsItems.GetOrAdd(monitoredItem, static _ => new AllEventsItemState());
ReconcileAllEventsMonitoring();
}
@@ -5488,8 +5558,9 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
var scopes = AclEnforced ? ConditionVisibilityScopes(node) : null;
var reportingSession = (context as ISessionSystemContext)?.SessionId;
var deferred = e is ConditionEventSnapshot { Deferred: true } d ? d : null;
foreach (var item in _allEventsItems.Keys)
foreach (var (item, state) in _allEventsItems)
{
if (e is AuditEventState && !IsAuditDeliverableTo(item)) continue;
if (scopes is not null && !IsConditionVisibleTo(item, scopes)) continue;
@@ -5501,10 +5572,36 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
continue;
}
item.QueueEvent(e);
if (deferred is null)
{
item.QueueEvent(e);
continue;
}
// #854: a deferred event reaches this item off Lock, so a ConditionRefresh of the item may have replayed a
// newer state since it was reported. Decided and queued under the item's gate, which the refresh holds while
// it queues its replay and installs its cut — so it lands either before the replay or not at all.
lock (state.Gate)
{
if (state.Cut?.Supersedes(deferred) == true)
{
Interlocked.Increment(ref _refreshSupersededDeliveries);
OtOpcUaTelemetry.ConditionEventDeliveriesSuperseded.Add(1);
continue;
}
DeferredServerObjectDeliveryProbe?.Invoke();
item.QueueEvent(e);
}
}
}
/// <summary>
/// Test probe (#854): runs on the drain thread between a deferred event's supersession check and its queueing to a
/// Server-object item, inside the item's gate — the window a refresh must not slip into. Null in production.
/// </summary>
internal Action? DeferredServerObjectDeliveryProbe { get; set; }
/// <summary>
/// The SDK's audit-event guard from <c>MonitoredNode2.OnReportEvent</c>, reproduced exactly: an audit
/// event reaches an item only while the server is auditing, and only over a SignAndEncrypt or
@@ -6068,19 +6165,15 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
}
// A refresh on a Server-object item is the other way it learns about conditions — it is how a
// client connecting after a transition gets the current alarm picture — and the SDK base replays
// every condition under every root notifier to it after only its own role-permission check. With
// enforcement on, those items are refreshed here instead, through the same
// IsConditionVisibleTo decision live delivery uses; an event that names no
// condition is withheld. With it off, the base runs unchanged (its per-notifier repeats of one
// condition share an EventId, which the item's queue already collapses to one copy).
// Items on one of this manager's own notifiers keep the base behaviour: a stood-down item is off
// its notifier's fan-out, which the base already treats as "not refreshed".
// client connecting after a transition gets the current alarm picture. Those items are always refreshed
// here (RefreshServerObjectItems): with enforcement on through the same IsConditionVisibleTo decision live
// delivery uses, and in both modes so the refresh can cut off deferred events it supersedes (#854).
// Items on one of this manager's own notifiers keep the base behaviour with enforcement off (a stood-down
// item is off its notifier's fan-out, which the base already treats as "not refreshed"), and
// RefreshDirectItems with it on.
/// <inheritdoc />
public override ServiceResult ConditionRefresh(OperationContext context, IList<IEventMonitoredItem> monitoredItems)
{
if (!AclEnforced) return base.ConditionRefresh(context, monitoredItems);
var direct = new List<IEventMonitoredItem>();
var serverItems = new List<IEventMonitoredItem>();
foreach (var item in monitoredItems)
@@ -6089,36 +6182,116 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2, INodeManager3
(item.NodeId == ObjectIds.Server ? serverItems : direct).Add(item);
}
var enforced = AclEnforced;
var systemContext = SystemContext.Copy(context);
if (direct.Count > 0) RefreshDirectItems(systemContext, direct);
if (serverItems.Count == 0) return ServiceResult.Good;
List<NodeState> notifiers;
lock (Lock) notifiers = RootNotifiers?.ToList() ?? [];
var events = new List<IFilterTarget>();
foreach (var notifier in notifiers) notifier.ConditionRefresh(systemContext, events, true);
// One snapshot per condition, with its scopes resolved once.
var conditions = new List<(IFilterTarget Event, List<NodeId> Scopes)>();
var seen = new HashSet<NodeState>(System.Collections.Generic.ReferenceEqualityComparer.Instance);
foreach (var e in events)
if (direct.Count > 0)
{
if (ConditionOf(e) is not { } condition || !seen.Add(condition)) continue;
conditions.Add((e, ConditionVisibilityScopes(condition)));
if (enforced) RefreshDirectItems(systemContext, direct);
else base.ConditionRefresh(context, direct);
}
foreach (var item in serverItems)
if (serverItems.Count > 0) RefreshServerObjectItems(systemContext, serverItems, enforced);
return ServiceResult.Good;
}
/// <summary>
/// ConditionRefresh (and ConditionRefresh2) for Server-object (<c>i=2253</c>) items: each condition under a root
/// notifier once, through the item's ACL decision when <paramref name="enforced"/> (#586) — and never overtaken by
/// a deferred event older than what it replays (Gitea #854).
/// </summary>
/// <param name="systemContext">The refresh's system context.</param>
/// <param name="items">The Server-object items being refreshed.</param>
/// <param name="enforced">Whether the data-plane ACL is enforced.</param>
/// <remarks>
/// <para>
/// <b>The overtaking (#854).</b> While a folder subscriber's channel is full, a condition's later events wait
/// in its deferral queue (#838) and reach Server-object items only when the drain gets to them, off
/// <c>Lock</c>. A refresh in that window replays the condition's CURRENT state between RefreshStart and
/// RefreshEnd; the waiting events, all older, then followed RefreshEnd — a stale state, or a condition the
/// refresh had just dropped re-added, until the newest drained.
/// </para>
/// <para>
/// <b>The cut.</b> Everything — the watermark (<see cref="_conditionEventSequence"/>), whether any event is
/// deferred, the replay itself and its queueing — happens in ONE hold of <c>Lock</c>, under which every condition
/// event is reported. So the replay is exactly the state at the watermark: an event at or below it is not newer
/// than the replay, one above it was reported after it. Each item's replay is queued and its
/// <see cref="RefreshCut"/> installed under the item's <see cref="AllEventsItemState.Gate"/>, which a deferred
/// delivery to the item also holds (<see cref="OnConditionReportEvent"/>): a deferred event therefore lands
/// before the replay, or is judged against the cut — withheld if superseded (<see cref="RefreshCut.Supersedes"/>).
/// Only events marked <see cref="ConditionEventSnapshot.Deferred"/> are judged: any other was delivered to
/// Server-object items synchronously under <c>Lock</c>, so it cannot follow a replay it predates.
/// </para>
/// <para>
/// <b>Per item, never global.</b> Only the refreshed item stops receiving the superseded events; the deferral
/// queue, every other Server-object item and every folder item still get them, in order. The refresh never
/// waits for a drain. No cut is kept while nothing is deferred (the common case).
/// </para>
/// <para>
/// <b>Lock order</b> is <c>Lock</c> → item gate → the SDK item's own lock, the same as a synchronous report
/// (<c>Lock</c> → <see cref="OnConditionReportEvent"/>); the drain takes the item gate without <c>Lock</c>, and
/// nothing under an item gate takes <c>Lock</c> or the event-report gate.
/// </para>
/// <para>
/// With enforcement off this replaces the SDK base's Server-object loop with the same result: the base queued
/// one copy per root notifier a condition is wired to, which the item's queue collapsed to one (they share an
/// EventId); here it is one per condition. A replayed event that names no condition is withheld only under
/// enforcement (fail closed), as before.
/// </para>
/// </remarks>
private void RefreshServerObjectItems(ServerSystemContext systemContext, List<IEventMonitoredItem> items, bool enforced)
{
lock (Lock)
{
foreach (var (e, scopes) in conditions)
var watermark = Interlocked.Read(ref _conditionEventSequence);
bool deferring;
lock (_eventReportGate) deferring = _deferredEventReports > 0;
var events = new List<IFilterTarget>();
foreach (var notifier in RootNotifiers?.ToList() ?? []) notifier.ConditionRefresh(systemContext, events, true);
// One event per condition, with its scopes resolved once.
var replay = new List<(IFilterTarget Event, NodeState? Condition, List<NodeId>? Scopes)>();
var seen = new HashSet<NodeState>(System.Collections.Generic.ReferenceEqualityComparer.Instance);
foreach (var e in events)
{
if (!IsConditionVisibleTo(item, scopes)) continue;
if (ServiceResult.IsBad(ValidateEventRolePermissions(item, e))) continue;
item.QueueEvent(e);
if (ConditionOf(e) is not { } condition)
{
if (!enforced) replay.Add((e, null, null));
continue;
}
if (!seen.Add(condition)) continue;
replay.Add((e, condition, enforced ? ConditionVisibilityScopes(condition) : null));
}
foreach (var item in items)
{
var queued = new List<IFilterTarget>();
var refreshed = new HashSet<NodeId>();
foreach (var (e, condition, scopes) in replay)
{
if (scopes is not null && !IsConditionVisibleTo(item, scopes)) continue;
if (ServiceResult.IsBad(ValidateEventRolePermissions(item, e))) continue;
queued.Add(e);
if (condition is not null) refreshed.Add(condition.NodeId);
}
if (!_allEventsItems.TryGetValue(item, out var state))
{
// Not (or no longer) subscribed: no deferred delivery can reach it, so there is nothing to cut.
foreach (var e in queued) item.QueueEvent(e);
continue;
}
lock (state.Gate)
{
foreach (var e in queued) item.QueueEvent(e);
// A cut is only needed while an event at or below the watermark may still be delivered. Refreshes are
// serialised by Lock, so a later one always carries the later watermark and simply replaces it.
state.Cut = deferring ? new RefreshCut(watermark, refreshed) : null;
}
}
}
return ServiceResult.Good;
}
/// <summary>