Files
lmxopcua/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs
T
Joseph Doherty d08e900203 fix(galaxy): #859 second review + #861 — window-sequence trim, per-session handles, straddling subscribes
I-2: unclaimed parked events are trimmed by window sequence. Each window gets a
monotonically increasing sequence at Begin, each parked event is stamped with
the newest one, and on every End events stamped before the oldest open
window's sequence are dropped (amortised O(1) via a park-order queue). Under
concurrent reads the open count rarely hit 0, so trailing events used to pile
up to the bound and push genuine early values into overflow.

I-1 / #861: SubscriptionRegistry.StartNewSession() — called FIRST in
ReplayAsync on every reconnect — clears held events, invalidates the reverse
map (subscriptions kept) and bumps a session number. New-session events whose
number an old binding still held now park until the replay's per-subscription
Rebind promotes them, instead of resolving to the wrong tag. TryRegister /
TryRebind refuse a stale session atomically under the park gate: a
SubscribeAsync whose reply straddled a reconnect is re-issued on the new
session (max 3 attempts), a subscribe-once read returns BadCommunicationError,
ResubscribeAllAsync stops if the session changes again. Old-session handle
numbers are never unsubscribed (Remove reports whether the entry belongs to
the current session; the read's late-reply path checks too), and an old
session's pump can no longer resolve through the map (fast path checks the
session).

I-3 (interim): a completed UnsubscribeBulk purges events parked for its
handles (driver + probe watcher). Full fix needs a worker sequence on the
SubscribeBulk reply (mxaccessgw, optional) — documented.

M-1: a resolved handle with a stranded parked list promotes it and resolves
the live event (never parks it). M-2: ResolveOrPark returns the ready events
to replay first, one gate acquisition per event while anything is held.
M-3: SubscribeAsync/read dispose the window before the orphan-release RPC.
M-5: ResolveOrPark remarks document the single-dispatching-pump invariant.

Tests (TestWait): overlapping windows don't accumulate, purge, stranded list,
session fence + map invalidation, stale-session registration refused,
end-to-end replay with swapped handle numbers and event-before-reply,
subscribe and read straddling a reconnect, read deadline closes its window.
Each was checked red against a mutation of the fix it pins.

Refs #859, #861.

Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
2026-11-11 10:41:16 -05:00

483 lines
24 KiB
C#

using System.Diagnostics.Metrics;
using System.Threading.Channels;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
/// <summary>
/// Long-running consumer of <see cref="IGalaxySubscriber.StreamEventsAsync"/>. Translates
/// each <see cref="MxEvent"/> with family <see cref="MxEventFamily.OnDataChange"/> into
/// <see cref="DataChangeEventArgs"/> and dispatches one event per registered driver
/// subscription that includes the changed item handle (fan-out via
/// <see cref="SubscriptionRegistry.ResolveSubscribers"/>).
/// </summary>
/// <remarks>
/// <para>
/// One pump per connected <see cref="GalaxyMxSession"/>. Reconnect lives in the
/// <see cref="ReconnectSupervisor"/>; on transport failure here we log and invoke
/// the optional <c>onStreamFault</c> callback so the owner (GalaxyDriver) can
/// forward the fault to <see cref="ReconnectSupervisor.ReportTransportFailure"/>
/// and the supervisor can drive reopen → replay.
/// </para>
/// <para>
/// PR 6.2 — the network-read loop and the listener-fanout loop are decoupled by a
/// bounded <see cref="Channel{T}"/>. When a listener is slow enough to fill the
/// channel, new events are dropped (newest-dropped semantics: producer's
/// <c>TryWrite</c> fails) rather than back-pressuring the gw stream. Three counters
/// on the <c>ZB.MOM.WW.OtOpcUa.Driver.Galaxy</c> meter expose received / dispatched
/// / dropped totals so ops sees pressure before it manifests as user-visible loss.
/// </para>
/// </remarks>
internal sealed class EventPump : IAsyncDisposable
{
public const string MeterName = "ZB.MOM.WW.OtOpcUa.Driver.Galaxy";
private const int DefaultChannelCapacity = 50_000;
// Single static meter so a host-level MeterListener catches all pump instances.
private static readonly Meter Meter = new(MeterName);
private static readonly Counter<long> EventsReceived =
Meter.CreateCounter<long>("galaxy.events.received", unit: "{event}",
description: "MxEvents read from the gateway StreamEvents stream.");
private static readonly Counter<long> EventsDispatched =
Meter.CreateCounter<long>("galaxy.events.dispatched", unit: "{event}",
description: "MxEvents passed through the bounded channel and into OnDataChange.");
private static readonly Counter<long> EventsDropped =
Meter.CreateCounter<long>("galaxy.events.dropped", unit: "{event}",
description: "MxEvents dropped because the bounded channel was full (newest-dropped).");
private static readonly Counter<long> ReplayGapsObserved =
Meter.CreateCounter<long>("galaxy.events.replay_gaps", unit: "{gap}",
description: "Replay-gap sentinels received — the gateway evicted events we can never see.");
private static readonly Counter<long> ValuesUndecodable =
Meter.CreateCounter<long>("galaxy.events.undecodable", unit: "{event}",
description: "Data-change events carrying a value kind this driver cannot represent.");
private static readonly Counter<long> EarlyEventsParked =
Meter.CreateCounter<long>("galaxy.events.early_parked", unit: "{event}",
description: "Data-change events that arrived before their SubscribeBulk reply registered the item handle, parked for replay (#859).");
private static readonly Counter<long> EarlyEventsReplayed =
Meter.CreateCounter<long>("galaxy.events.early_replayed", unit: "{event}",
description: "Parked early data-change events replayed once their item handle was registered (#859).");
private static readonly Counter<long> EarlyEventsOverflowed =
Meter.CreateCounter<long>("galaxy.events.early_overflow", unit: "{event}",
description: "Early data-change events dropped because the parking bound was reached (#859).");
private readonly IGalaxySubscriber _subscriber;
private readonly SubscriptionRegistry _registry;
private readonly ILogger _logger;
private readonly Func<long, ISubscriptionHandle> _handleFactory;
private readonly Action<Exception>? _onStreamFault;
private readonly Action<ReplayGap>? _onReplayGap;
private readonly Channel<MxEvent> _channel;
private readonly KeyValuePair<string, object?> _clientTag;
private readonly CancellationTokenSource _cts = new();
private Task? _loop;
private Task? _dispatchLoop;
private bool _disposed;
/// <summary>
/// Value kinds already warned about. Bounded by the <c>MxValue</c> oneof enum, so a
/// broken producer cannot flood the log, while a SECOND unrepresentable kind appearing
/// later still gets its own line — warn-once-per-pump would have swallowed it.
/// </summary>
private readonly HashSet<MxValue.KindOneofCase> _undecodableKindsWarned = [];
/// <summary>Occurs when a data change event is received from the Galaxy subscriber.</summary>
public event EventHandler<DataChangeEventArgs>? OnDataChange;
/// <summary>Initializes a new instance of the EventPump class.</summary>
/// <param name="subscriber">The Galaxy subscriber to consume events from.</param>
/// <param name="registry">The subscription registry for resolving subscribers.</param>
/// <param name="logger">The logger instance; if null, uses NullLogger.</param>
/// <param name="handleFactory">The factory for creating subscription handles; if null, uses GalaxySubscriptionHandle.</param>
/// <param name="channelCapacity">The bounded channel capacity for buffering events.</param>
/// <param name="clientName">The client name for metric tagging; if null, uses "&lt;unknown&gt;".</param>
/// <param name="onStreamFault">Optional callback invoked when the stream faults.</param>
/// <param name="onReplayGap">
/// Optional callback invoked when the gateway reports a replay gap. The contract
/// obliges the client to re-snapshot; see <see cref="HandleReplayGap"/>.
/// </param>
public EventPump(
IGalaxySubscriber subscriber,
SubscriptionRegistry registry,
ILogger? logger = null,
Func<long, ISubscriptionHandle>? handleFactory = null,
int channelCapacity = DefaultChannelCapacity,
string? clientName = null,
Action<Exception>? onStreamFault = null,
Action<ReplayGap>? onReplayGap = null)
{
_subscriber = subscriber ?? throw new ArgumentNullException(nameof(subscriber));
_registry = registry ?? throw new ArgumentNullException(nameof(registry));
_logger = logger ?? NullLogger.Instance;
_handleFactory = handleFactory ?? (id => new GalaxySubscriptionHandle(id));
_onStreamFault = onStreamFault;
_onReplayGap = onReplayGap;
if (channelCapacity < 1)
{
throw new ArgumentOutOfRangeException(nameof(channelCapacity),
"channelCapacity must be >= 1; recommended 50_000 for 50k-tag deployments.");
}
_channel = Channel.CreateBounded<MxEvent>(new BoundedChannelOptions(channelCapacity)
{
// Newest-dropped semantics: we use FullMode.Wait but never call the
// awaitable WriteAsync — only the synchronous TryWrite below in
// RunAsync. With Wait + TryWrite, a full channel makes TryWrite return
// false immediately, which we account for via the EventsDropped counter.
// We deliberately do NOT use BoundedChannelFullMode.DropWrite — that
// would silently discard the new event inside Channel<T> without
// surfacing the drop on a counter (keep this comment and the FullMode
// value consistent).
FullMode = BoundedChannelFullMode.Wait,
SingleReader = true,
// Not single-writer: besides the stream read loop, a registration that promotes
// parked early events (#859) writes the WakeSentinel from the subscribing thread.
SingleWriter = false,
});
_clientTag = new KeyValuePair<string, object?>("galaxy.client", clientName ?? "<unknown>");
// One pump per gateway session. The owner calls SubscriptionRegistry.StartNewSession() BEFORE
// creating the pump for a new session (reconnect); this pump binds to that session and never
// parks or replays under any other (#859 review — item handles are reused across sessions).
_session = _registry.CurrentSession;
_registry.ParkedEventsReady += WakeForReadyEvents;
}
private readonly int _session;
/// <summary>
/// Marker written into the channel to wake the dispatch loop after a registration promoted
/// parked early events (#859). Never dispatched itself; compared by reference.
/// </summary>
private static readonly MxEvent WakeSentinel = new();
/// <summary>
/// Wakes the dispatch loop so promoted early events replay even if no further event arrives.
/// Runs on the registering thread, possibly under the driver's <c>_registryGate</c> or the probe
/// watcher's <c>_syncLock</c> — safe because the channel keeps the default
/// <c>AllowSynchronousContinuations = false</c>, so the write never runs the dispatch loop inline.
/// A failed write is harmless: the channel is either full (the loop is busy and replays the
/// ready queue before its next dispatch) or completed (the pump is dead; its held events
/// belong to a session the next StartNewSession discards).
/// </summary>
private void WakeForReadyEvents() => _channel.Writer.TryWrite(WakeSentinel);
/// <summary>
/// Start consuming the event stream on a background task. Idempotent — second
/// calls are no-ops while the loop is running.
/// </summary>
public void Start()
{
ObjectDisposedException.ThrowIf(_disposed, this);
if (_loop is not null) return;
_loop = Task.Run(() => RunAsync(_cts.Token));
_dispatchLoop = Task.Run(() => DispatchLoopAsync(_cts.Token));
}
private async Task RunAsync(CancellationToken ct)
{
try
{
await foreach (var ev in _subscriber.StreamEventsAsync(ct).WithCancellation(ct).ConfigureAwait(false))
{
if (ct.IsCancellationRequested) break;
EventsReceived.Add(1, _clientTag);
// The replay-gap sentinel is a control signal, not data, and is handled HERE
// rather than in the dispatch loop for one reason: everything past this point
// is subject to newest-dropped channel pressure, and a dropped "you have
// missed events" signal is the one loss that cannot be recovered from later.
// It carries no item payload, so there is nothing to fan out downstream.
if (ev.ReplayGap is { } gap)
{
HandleReplayGap(gap);
continue;
}
// Newest-dropped: TryWrite fast-paths the common case (channel has room).
// When full we count the drop and continue reading the gw stream so
// back-pressure doesn't propagate upstream.
if (!_channel.Writer.TryWrite(ev))
{
EventsDropped.Add(1, _clientTag);
}
}
// Falling out of the loop without an exception means the SERVER completed the
// stream — a closed or evicted gateway session, not a transport failure. That is
// just as fatal to this driver as a fault (the pump is now dead and no further
// data-change events can ever arrive) but it raises nothing, so without this the
// pump exits silently, the supervisor is never told, GetHealth() keeps reporting
// Healthy, and the comms-loss watchdog never trips either — it requires a
// non-Healthy state. The result is a driver that is permanently, invisibly deaf.
if (!ct.IsCancellationRequested)
{
var cause = new InvalidOperationException(
"The gateway completed the StreamEvents stream without an error. The session was " +
"closed or evicted server-side; the event pump cannot receive further events.");
_logger.LogWarning(
"Galaxy EventPump stream was completed by the gateway — notifying reconnect supervisor.");
NotifyStreamFault(cause);
}
}
catch (Exception) when (ct.IsCancellationRequested)
{
// Clean shutdown — no log. A cancelled gRPC stream may surface as RpcException(Cancelled)
// rather than OperationCanceledException; either way the pump was told to stop, so this
// is not a transport fault and must not wake the reconnect supervisor mid-teardown.
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"Galaxy EventPump loop ended with an exception — notifying reconnect supervisor.");
// The gw StreamEvents stream faulted. Signal the reconnect supervisor so it
// drives reopen → replay. Without this the stream silently dies and a
// transient gateway drop permanently stops data-change notifications.
NotifyStreamFault(ex);
}
finally
{
// Tell the dispatch loop the producer is done so it drains and exits.
_channel.Writer.TryComplete();
}
}
/// <summary>
/// Logs the first occurrence of each unrepresentable value kind. The dispatch loop is
/// single-reader, so the set needs no lock — but it is only ever touched from there.
/// </summary>
private void WarnUndecodableOnce(MxEvent ev)
{
var kind = ev.Value?.KindCase ?? MxValue.KindOneofCase.None;
if (!_undecodableKindsWarned.Add(kind)) return;
_logger.LogWarning(
"Galaxy event for item handle {ItemHandle} carried a value kind this driver cannot " +
"represent ({KindCase}) — publishing BadTypeMismatch. This means the gateway is sending " +
"a wire type MxValueDecoder has no mapping for; readings of this kind are being lost. " +
"Further occurrences of {KindCase} are counted on galaxy.events.undecodable, not logged.",
ev.ItemHandle, kind, kind);
}
private void NotifyStreamFault(Exception cause)
{
if (_onStreamFault is null) return;
try { _onStreamFault(cause); }
catch (Exception cbEx)
{
_logger.LogWarning(cbEx,
"Galaxy EventPump stream-fault callback threw — supervisor may not have been notified.");
}
}
/// <summary>
/// Handles the gateway's replay-gap sentinel: events between the sequence we asked to
/// resume after and the oldest the gateway still retains were evicted and can never be
/// delivered. The contract is explicit that a client "MUST treat a present replay_gap as
/// 'you missed events — discard local state and re-snapshot'".
/// </summary>
/// <remarks>
/// ⚠️ <b>This is unreachable in the current configuration and that is deliberate.</b>
/// The gateway emits the sentinel only on a stream opened with
/// <c>after_worker_sequence &gt; 0</c>; <c>GatewayGalaxySubscriber</c> passes 0, which the
/// gateway treats as "fresh stream, not a resume: plain attach, no replay, no sentinel".
/// It is implemented anyway because the alternative — the sentinel falling through the
/// family switch into the default drop, which is what happened before, since a sentinel
/// carries <c>family = UNSPECIFIED</c> — means silently ignoring a documented MUST if
/// resume is ever switched on. Enabling resume is a larger change than it looks: the
/// sequence is only meaningful within one gateway session, and our reconnect path
/// recreates the session, so a remembered sequence replayed against a fresh session
/// would set the server's live watermark above every event it is about to send.
/// </remarks>
private void HandleReplayGap(ReplayGap gap)
{
ReplayGapsObserved.Add(1, _clientTag);
_logger.LogWarning(
"Galaxy event stream reported a replay gap: events after worker_sequence {RequestedAfter} " +
"up to {OldestAvailable} were evicted from the gateway's replay ring and cannot be " +
"recovered. Forcing a re-snapshot of every active subscription.",
gap.RequestedAfterSequence, gap.OldestAvailableSequence);
if (_onReplayGap is null) return;
try { _onReplayGap(gap); }
catch (Exception ex)
{
_logger.LogWarning(ex,
"Galaxy EventPump replay-gap callback threw — the re-snapshot may not have run.");
}
}
private async Task DispatchLoopAsync(CancellationToken ct)
{
try
{
await foreach (var ev in _channel.Reader.ReadAllAsync(ct).ConfigureAwait(false))
{
if (ReferenceEquals(ev, WakeSentinel))
{
DispatchReadyEvents();
continue;
}
Dispatch(ev);
EventsDispatched.Add(1, _clientTag);
}
}
catch (Exception) when (ct.IsCancellationRequested)
{
// Clean shutdown.
}
catch (Exception ex)
{
// The read loop is still draining the stream into a channel nobody reads: the pump is
// as dead as after a transport fault, but used to raise nothing — health stayed
// Healthy and the comms-loss watchdog never tripped (a clean termination evading the
// watchdog). Report it as a stream fault so the supervisor recreates the pump.
_logger.LogWarning(ex,
"Galaxy EventPump dispatch loop ended with an exception — notifying reconnect supervisor so the pump is recreated.");
NotifyStreamFault(ex);
}
}
private void Dispatch(MxEvent ev)
{
switch (ev.Family)
{
case MxEventFamily.OnDataChange:
DispatchDataChange(ev);
break;
default:
// OnAlarmTransition is no longer carried on the per-session event stream
// — alarms come from the gateway's session-less StreamAlarms feed
// (GatewayGalaxyAlarmFeed). OnWriteComplete / OperationComplete /
// OnBufferedDataChange are filtered out: write callers get their reply
// via the InvokeAsync round-trip, not via the event stream. Still replay any
// promoted early events: a wake lost to a full channel relies on the next item.
DispatchReadyEvents();
return;
}
}
private void DispatchDataChange(MxEvent ev)
{
var outcome = _registry.ResolveOrPark(ev, _session, out var subscribers, out var replayFirst);
// Replay promoted early events (#859) BEFORE fanning this event out. A registration writes
// the reverse map and promotes its parked events in ONE critical section under the
// registry's park gate, and ResolveOrPark resolves — and takes the ready queue — under that
// same gate whenever anything is held. So if this event resolved, every older parked event
// for its handle is in replayFirst, and replaying it first preserves per-handle order.
ReplayEarly(replayFirst);
switch (outcome)
{
case EarlyEventOutcome.Resolved:
break;
case EarlyEventOutcome.Parked:
// The handle's SubscribeBulk reply has not been registered yet — MXAccess's initial
// value can overtake it. Replayed when the registration lands.
EarlyEventsParked.Add(1, _clientTag);
return;
case EarlyEventOutcome.Overflow:
EarlyEventsOverflowed.Add(1, _clientTag);
return;
default:
return; // stale event after unsubscribe (no subscribe in flight) — drop quietly
}
FanOut(ev, subscribers);
}
/// <summary>
/// Dispatch every early event a registration promoted (#859). Runs only on the dispatch
/// thread, so it is serialised with live dispatch. A promoted event whose subscription was
/// removed again before the replay resolves to nobody and is dropped like any stale event.
/// </summary>
private void DispatchReadyEvents() => ReplayEarly(_registry.TakeReadyEvents(_session));
private void ReplayEarly(IReadOnlyList<MxEvent> ready)
{
if (ready.Count == 0) return;
foreach (var early in ready)
{
var subscribers = _registry.ResolveSubscribers(early.ItemHandle);
if (subscribers.Count == 0) continue;
EarlyEventsReplayed.Add(1, _clientTag);
FanOut(early, subscribers);
}
}
private void FanOut(MxEvent ev, IReadOnlyList<(long SubscriptionId, string FullReference)> subscribers)
{
var snapshot = ToSnapshot(ev);
foreach (var (subscriptionId, fullReference) in subscribers)
{
var handle = _handleFactory(subscriptionId);
try
{
OnDataChange?.Invoke(this, new DataChangeEventArgs(handle, fullReference, snapshot));
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"Galaxy OnDataChange handler threw for {FullRef} subscription {SubscriptionId} — continuing fan-out.",
fullReference, subscriptionId);
}
}
}
private DataValueSnapshot ToSnapshot(MxEvent ev)
{
if (!MxValueDecoder.TryDecode(ev.Value, out var value))
{
// A value we cannot represent is published as Bad, NOT as a null carrying the
// event's own (typically Good) quality. A good null is indistinguishable from a
// real null reading, so a client would treat a value it never received as a
// genuine one — the failure would be invisible on the wire and survive only in a
// log nobody is reading. BadTypeMismatch says what actually happened: the payload
// arrived, and this driver has no CLR type to carry it in.
ValuesUndecodable.Add(1, _clientTag);
WarnUndecodableOnce(ev);
return new DataValueSnapshot(
Value: null,
StatusCode: StatusCodeMap.BadTypeMismatch,
SourceTimestampUtc: ev.SourceTimestamp is { } undecodableTs ? undecodableTs.ToDateTime() : null,
ServerTimestampUtc: DateTime.UtcNow);
}
var statusCode = ev.Statuses.Count > 0
? StatusCodeMap.FromMxStatus(ev.Statuses[0], _logger)
: StatusCodeMap.FromQualityByte((byte)(ev.Quality & 0xFF), _logger);
DateTime? sourceTimestamp = ev.SourceTimestamp is { } ts ? ts.ToDateTime() : null;
return new DataValueSnapshot(
Value: value,
StatusCode: statusCode,
SourceTimestampUtc: sourceTimestamp,
ServerTimestampUtc: DateTime.UtcNow);
}
/// <summary>Disposes the event pump and cancels all running tasks.</summary>
/// <returns>A value task representing the asynchronous disposal operation.</returns>
public async ValueTask DisposeAsync()
{
if (_disposed) return;
_disposed = true;
_registry.ParkedEventsReady -= WakeForReadyEvents;
_cts.Cancel();
_channel.Writer.TryComplete();
if (_loop is not null)
{
try { await _loop.ConfigureAwait(false); } catch { /* shutdown */ }
}
if (_dispatchLoop is not null)
{
try { await _dispatchLoop.ConfigureAwait(false); } catch { /* shutdown */ }
}
_cts.Dispose();
}
}