using System.Diagnostics.Metrics; using System.Threading.Channels; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using ZB.MOM.WW.MxGateway.Contracts.Proto; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime; /// /// Long-running consumer of . Translates /// each with family into /// and dispatches one event per registered driver /// subscription that includes the changed item handle (fan-out via /// ). /// /// /// /// One pump per connected . Reconnect lives in the /// ; on transport failure here we log and invoke /// the optional onStreamFault callback so the owner (GalaxyDriver) can /// forward the fault to /// and the supervisor can drive reopen → replay. /// /// /// PR 6.2 — the network-read loop and the listener-fanout loop are decoupled by a /// bounded . When a listener is slow enough to fill the /// channel, new events are dropped (newest-dropped semantics: producer's /// TryWrite fails) rather than back-pressuring the gw stream. Three counters /// on the ZB.MOM.WW.OtOpcUa.Driver.Galaxy meter expose received / dispatched /// / dropped totals so ops sees pressure before it manifests as user-visible loss. /// /// internal sealed class EventPump : IAsyncDisposable { public const string MeterName = "ZB.MOM.WW.OtOpcUa.Driver.Galaxy"; private const int DefaultChannelCapacity = 50_000; // Single static meter so a host-level MeterListener catches all pump instances. private static readonly Meter Meter = new(MeterName); private static readonly Counter EventsReceived = Meter.CreateCounter("galaxy.events.received", unit: "{event}", description: "MxEvents read from the gateway StreamEvents stream."); private static readonly Counter EventsDispatched = Meter.CreateCounter("galaxy.events.dispatched", unit: "{event}", description: "MxEvents passed through the bounded channel and into OnDataChange."); private static readonly Counter EventsDropped = Meter.CreateCounter("galaxy.events.dropped", unit: "{event}", description: "MxEvents dropped because the bounded channel was full (newest-dropped)."); private static readonly Counter ReplayGapsObserved = Meter.CreateCounter("galaxy.events.replay_gaps", unit: "{gap}", description: "Replay-gap sentinels received — the gateway evicted events we can never see."); private static readonly Counter ValuesUndecodable = Meter.CreateCounter("galaxy.events.undecodable", unit: "{event}", description: "Data-change events carrying a value kind this driver cannot represent."); private readonly IGalaxySubscriber _subscriber; private readonly SubscriptionRegistry _registry; private readonly ILogger _logger; private readonly Func _handleFactory; private readonly Action? _onStreamFault; private readonly Action? _onReplayGap; private readonly Channel _channel; private readonly KeyValuePair _clientTag; private readonly CancellationTokenSource _cts = new(); private Task? _loop; private Task? _dispatchLoop; private bool _disposed; /// /// Value kinds already warned about. Bounded by the MxValue oneof enum, so a /// broken producer cannot flood the log, while a SECOND unrepresentable kind appearing /// later still gets its own line — warn-once-per-pump would have swallowed it. /// private readonly HashSet _undecodableKindsWarned = []; /// Occurs when a data change event is received from the Galaxy subscriber. public event EventHandler? OnDataChange; /// Initializes a new instance of the EventPump class. /// The Galaxy subscriber to consume events from. /// The subscription registry for resolving subscribers. /// The logger instance; if null, uses NullLogger. /// The factory for creating subscription handles; if null, uses GalaxySubscriptionHandle. /// The bounded channel capacity for buffering events. /// The client name for metric tagging; if null, uses "<unknown>". /// Optional callback invoked when the stream faults. /// /// Optional callback invoked when the gateway reports a replay gap. The contract /// obliges the client to re-snapshot; see . /// public EventPump( IGalaxySubscriber subscriber, SubscriptionRegistry registry, ILogger? logger = null, Func? handleFactory = null, int channelCapacity = DefaultChannelCapacity, string? clientName = null, Action? onStreamFault = null, Action? onReplayGap = null) { _subscriber = subscriber ?? throw new ArgumentNullException(nameof(subscriber)); _registry = registry ?? throw new ArgumentNullException(nameof(registry)); _logger = logger ?? NullLogger.Instance; _handleFactory = handleFactory ?? (id => new GalaxySubscriptionHandle(id)); _onStreamFault = onStreamFault; _onReplayGap = onReplayGap; if (channelCapacity < 1) { throw new ArgumentOutOfRangeException(nameof(channelCapacity), "channelCapacity must be >= 1; recommended 50_000 for 50k-tag deployments."); } _channel = Channel.CreateBounded(new BoundedChannelOptions(channelCapacity) { // Newest-dropped semantics: we use FullMode.Wait but never call the // awaitable WriteAsync — only the synchronous TryWrite below in // RunAsync. With Wait + TryWrite, a full channel makes TryWrite return // false immediately, which we account for via the EventsDropped counter. // We deliberately do NOT use BoundedChannelFullMode.DropWrite — that // would silently discard the new event inside Channel without // surfacing the drop on a counter (keep this comment and the FullMode // value consistent). FullMode = BoundedChannelFullMode.Wait, SingleReader = true, SingleWriter = true, }); _clientTag = new KeyValuePair("galaxy.client", clientName ?? ""); } /// /// Start consuming the event stream on a background task. Idempotent — second /// calls are no-ops while the loop is running. /// public void Start() { ObjectDisposedException.ThrowIf(_disposed, this); if (_loop is not null) return; _loop = Task.Run(() => RunAsync(_cts.Token)); _dispatchLoop = Task.Run(() => DispatchLoopAsync(_cts.Token)); } private async Task RunAsync(CancellationToken ct) { try { await foreach (var ev in _subscriber.StreamEventsAsync(ct).WithCancellation(ct).ConfigureAwait(false)) { if (ct.IsCancellationRequested) break; EventsReceived.Add(1, _clientTag); // The replay-gap sentinel is a control signal, not data, and is handled HERE // rather than in the dispatch loop for one reason: everything past this point // is subject to newest-dropped channel pressure, and a dropped "you have // missed events" signal is the one loss that cannot be recovered from later. // It carries no item payload, so there is nothing to fan out downstream. if (ev.ReplayGap is { } gap) { HandleReplayGap(gap); continue; } // Newest-dropped: TryWrite fast-paths the common case (channel has room). // When full we count the drop and continue reading the gw stream so // back-pressure doesn't propagate upstream. if (!_channel.Writer.TryWrite(ev)) { EventsDropped.Add(1, _clientTag); } } // Falling out of the loop without an exception means the SERVER completed the // stream — a closed or evicted gateway session, not a transport failure. That is // just as fatal to this driver as a fault (the pump is now dead and no further // data-change events can ever arrive) but it raises nothing, so without this the // pump exits silently, the supervisor is never told, GetHealth() keeps reporting // Healthy, and the comms-loss watchdog never trips either — it requires a // non-Healthy state. The result is a driver that is permanently, invisibly deaf. if (!ct.IsCancellationRequested) { var cause = new InvalidOperationException( "The gateway completed the StreamEvents stream without an error. The session was " + "closed or evicted server-side; the event pump cannot receive further events."); _logger.LogWarning( "Galaxy EventPump stream was completed by the gateway — notifying reconnect supervisor."); NotifyStreamFault(cause); } } catch (Exception) when (ct.IsCancellationRequested) { // Clean shutdown — no log. A cancelled gRPC stream may surface as RpcException(Cancelled) // rather than OperationCanceledException; either way the pump was told to stop, so this // is not a transport fault and must not wake the reconnect supervisor mid-teardown. } catch (Exception ex) { _logger.LogWarning(ex, "Galaxy EventPump loop ended with an exception — notifying reconnect supervisor."); // The gw StreamEvents stream faulted. Signal the reconnect supervisor so it // drives reopen → replay. Without this the stream silently dies and a // transient gateway drop permanently stops data-change notifications. NotifyStreamFault(ex); } finally { // Tell the dispatch loop the producer is done so it drains and exits. _channel.Writer.TryComplete(); } } /// /// Logs the first occurrence of each unrepresentable value kind. The dispatch loop is /// single-reader, so the set needs no lock — but it is only ever touched from there. /// private void WarnUndecodableOnce(MxEvent ev) { var kind = ev.Value?.KindCase ?? MxValue.KindOneofCase.None; if (!_undecodableKindsWarned.Add(kind)) return; _logger.LogWarning( "Galaxy event for item handle {ItemHandle} carried a value kind this driver cannot " + "represent ({KindCase}) — publishing BadTypeMismatch. This means the gateway is sending " + "a wire type MxValueDecoder has no mapping for; readings of this kind are being lost. " + "Further occurrences of {KindCase} are counted on galaxy.events.undecodable, not logged.", ev.ItemHandle, kind, kind); } private void NotifyStreamFault(Exception cause) { if (_onStreamFault is null) return; try { _onStreamFault(cause); } catch (Exception cbEx) { _logger.LogWarning(cbEx, "Galaxy EventPump stream-fault callback threw — supervisor may not have been notified."); } } /// /// Handles the gateway's replay-gap sentinel: events between the sequence we asked to /// resume after and the oldest the gateway still retains were evicted and can never be /// delivered. The contract is explicit that a client "MUST treat a present replay_gap as /// 'you missed events — discard local state and re-snapshot'". /// /// /// ⚠️ This is unreachable in the current configuration and that is deliberate. /// The gateway emits the sentinel only on a stream opened with /// after_worker_sequence > 0; GatewayGalaxySubscriber passes 0, which the /// gateway treats as "fresh stream, not a resume: plain attach, no replay, no sentinel". /// It is implemented anyway because the alternative — the sentinel falling through the /// family switch into the default drop, which is what happened before, since a sentinel /// carries family = UNSPECIFIED — means silently ignoring a documented MUST if /// resume is ever switched on. Enabling resume is a larger change than it looks: the /// sequence is only meaningful within one gateway session, and our reconnect path /// recreates the session, so a remembered sequence replayed against a fresh session /// would set the server's live watermark above every event it is about to send. /// private void HandleReplayGap(ReplayGap gap) { ReplayGapsObserved.Add(1, _clientTag); _logger.LogWarning( "Galaxy event stream reported a replay gap: events after worker_sequence {RequestedAfter} " + "up to {OldestAvailable} were evicted from the gateway's replay ring and cannot be " + "recovered. Forcing a re-snapshot of every active subscription.", gap.RequestedAfterSequence, gap.OldestAvailableSequence); if (_onReplayGap is null) return; try { _onReplayGap(gap); } catch (Exception ex) { _logger.LogWarning(ex, "Galaxy EventPump replay-gap callback threw — the re-snapshot may not have run."); } } private async Task DispatchLoopAsync(CancellationToken ct) { try { await foreach (var ev in _channel.Reader.ReadAllAsync(ct).ConfigureAwait(false)) { Dispatch(ev); EventsDispatched.Add(1, _clientTag); } } catch (OperationCanceledException) when (ct.IsCancellationRequested) { // Clean shutdown. } catch (Exception ex) { _logger.LogWarning(ex, "Galaxy EventPump dispatch loop ended with an exception — events past this point will be lost until restart."); } } private void Dispatch(MxEvent ev) { switch (ev.Family) { case MxEventFamily.OnDataChange: DispatchDataChange(ev); break; default: // OnAlarmTransition is no longer carried on the per-session event stream // — alarms come from the gateway's session-less StreamAlarms feed // (GatewayGalaxyAlarmFeed). OnWriteComplete / OperationComplete / // OnBufferedDataChange are filtered out: write callers get their reply // via the InvokeAsync round-trip, not via the event stream. return; } } private void DispatchDataChange(MxEvent ev) { var subscribers = _registry.ResolveSubscribers(ev.ItemHandle); if (subscribers.Count == 0) return; // stale event after unsubscribe — drop quietly var snapshot = ToSnapshot(ev); foreach (var (subscriptionId, fullReference) in subscribers) { var handle = _handleFactory(subscriptionId); try { OnDataChange?.Invoke(this, new DataChangeEventArgs(handle, fullReference, snapshot)); } catch (Exception ex) { _logger.LogWarning(ex, "Galaxy OnDataChange handler threw for {FullRef} subscription {SubscriptionId} — continuing fan-out.", fullReference, subscriptionId); } } } private DataValueSnapshot ToSnapshot(MxEvent ev) { if (!MxValueDecoder.TryDecode(ev.Value, out var value)) { // A value we cannot represent is published as Bad, NOT as a null carrying the // event's own (typically Good) quality. A good null is indistinguishable from a // real null reading, so a client would treat a value it never received as a // genuine one — the failure would be invisible on the wire and survive only in a // log nobody is reading. BadTypeMismatch says what actually happened: the payload // arrived, and this driver has no CLR type to carry it in. ValuesUndecodable.Add(1, _clientTag); WarnUndecodableOnce(ev); return new DataValueSnapshot( Value: null, StatusCode: StatusCodeMap.BadTypeMismatch, SourceTimestampUtc: ev.SourceTimestamp is { } undecodableTs ? undecodableTs.ToDateTime() : null, ServerTimestampUtc: DateTime.UtcNow); } var statusCode = ev.Statuses.Count > 0 ? StatusCodeMap.FromMxStatus(ev.Statuses[0], _logger) : StatusCodeMap.FromQualityByte((byte)(ev.Quality & 0xFF), _logger); DateTime? sourceTimestamp = ev.SourceTimestamp is { } ts ? ts.ToDateTime() : null; return new DataValueSnapshot( Value: value, StatusCode: statusCode, SourceTimestampUtc: sourceTimestamp, ServerTimestampUtc: DateTime.UtcNow); } /// Disposes the event pump and cancels all running tasks. /// A value task representing the asynchronous disposal operation. public async ValueTask DisposeAsync() { if (_disposed) return; _disposed = true; _cts.Cancel(); _channel.Writer.TryComplete(); if (_loop is not null) { try { await _loop.ConfigureAwait(false); } catch { /* shutdown */ } } if (_dispatchLoop is not null) { try { await _dispatchLoop.ConfigureAwait(false); } catch { /* shutdown */ } } _cts.Dispose(); } }