fix(galaxy): park data-change events that overtake their SubscribeBulk reply (#859)

mxaccessgw gives no reply-before-event ordering: the worker completes the STA
command via a RunContinuationsAsynchronously TCS, so the reply is framed on a
thread-pool continuation while the STA already pumps the OnDataChange Advise
queued (drained/framed by a separate loop), and the reply (unary Invoke) and
the event (StreamEvents) are separate HTTP/2 streams. The driver then also
registers handles concurrently with the pump's dispatch loop. MXAccess's
initial value could therefore be dropped as "stale after unsubscribe",
leaving the node at its pre-subscribe value until the next change.

SubscriptionRegistry gains an early-event window: every SubscribeBulk call
site (SubscribeAsync, ResubscribeAllAsync, PerPlatformProbeWatcher) opens
BeginPendingSubscribe() before the RPC and closes it after registering.
Inside the window unknown-handle events are parked (bounded, 100k); the
Register/Rebind that binds a handle promotes them, and the EventPump replays
them on its single dispatch thread before any later event for that handle
(woken by a sentinel when no further event arrives). Closing the last window
discards unclaimed events, so the stale drop is unchanged outside it.
New counters galaxy.events.early_{parked,replayed,overflow}.

Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs

@@ -253,6 +253,9 @@ public sealed class GalaxyDriver
     /// <summary>Test seam: the number of driver subscriptions the registry still tracks.</summary>
     internal int TrackedSubscriptionCountForTests => _subscriptions.TrackedSubscriptionCount;
 
+    /// <summary>Test seam — the registry's parked early-event count (#859).</summary>
+    internal int ParkedEarlyEventCountForTests => _subscriptions.ParkedEventCount;
+
     /// <summary>
     ///     Server-pushed data-change notification. Fires from the
     ///     <see cref="EventPump"/>'s background loop; handlers should be cheap (or queue
@@ -993,6 +996,10 @@ public sealed class GalaxyDriver
         CancellationToken cancellationToken)
     {
         var tagCount = 0;
+
+        // #859: the re-issued handles' initial values (the whole point of a re-snapshot) can
+        // overtake each SubscribeBulk reply; park them until the matching Rebind promotes them.
+        using var earlyEventWindow = _subscriptions.BeginPendingSubscribe();
         foreach (var (subscriptionId, oldBindings) in entries)
         {
             if (Volatile.Read(ref _runtimeGeneration) != generation)
@@ -1646,6 +1653,13 @@ public sealed class GalaxyDriver
         // and infrastructure callers (probe watcher, deploy watcher) hit the fallback.
         var requested = (int)Math.Max(0, publishingInterval.TotalMilliseconds);
         var bufferedIntervalMs = requested > 0 ? requested : _options.MxAccess.PublishingIntervalMs;
+
+        // #859: MXAccess's initial value for a new handle can reach the event pump before this
+        // reply is registered (the gateway gives no reply-before-event ordering, and the reply and
+        // the event ride separate RPCs). Open the early-event window BEFORE the RPC so such events
+        // are parked rather than dropped as stale, and close it only after TryRegisterForGeneration
+        // (method scope), which promotes them for in-order replay.
+        using var earlyEventWindow = _subscriptions.BeginPendingSubscribe();
         var results = await subscriber
             .SubscribeBulkAsync(dialledRefs, bufferedIntervalMs, cancellationToken)
             .ConfigureAwait(false);

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Health/PerPlatformProbeWatcher.cs

@@ -138,6 +138,10 @@ public sealed class PerPlatformProbeWatcher : IDisposable
     private async Task SubscribeAndRecordAsync(IReadOnlyList<string> platforms, CancellationToken cancellationToken)
     {
         var probeAddresses = platforms.Select(p => p + ProbeSuffix).ToArray();
+
+        // #859: the initial ScanState value can overtake the SubscribeBulk reply; park it until
+        // PublishBindingsLocked registers the handles below (window closes at method exit).
+        using var earlyEventWindow = _registry?.BeginPendingSubscribe();
         // PR 6.3 — use the configured bufferedUpdateIntervalMs (defaults to 0 = gw cadence
         // when the driver hasn't overridden MxAccess.PublishingIntervalMs). Probe ScanState
         // changes are rare so a coarser interval is usually fine; deployments that need

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs

@@ -53,6 +53,15 @@ internal sealed class EventPump : IAsyncDisposable
     private static readonly Counter<long> ValuesUndecodable =
         Meter.CreateCounter<long>("galaxy.events.undecodable", unit: "{event}",
             description: "Data-change events carrying a value kind this driver cannot represent.");
+    private static readonly Counter<long> EarlyEventsParked =
+        Meter.CreateCounter<long>("galaxy.events.early_parked", unit: "{event}",
+            description: "Data-change events that arrived before their SubscribeBulk reply registered the item handle, parked for replay (#859).");
+    private static readonly Counter<long> EarlyEventsReplayed =
+        Meter.CreateCounter<long>("galaxy.events.early_replayed", unit: "{event}",
+            description: "Parked early data-change events replayed once their item handle was registered (#859).");
+    private static readonly Counter<long> EarlyEventsOverflowed =
+        Meter.CreateCounter<long>("galaxy.events.early_overflow", unit: "{event}",
+            description: "Early data-change events dropped because the parking bound was reached (#859).");
 
     private readonly IGalaxySubscriber _subscriber;
     private readonly SubscriptionRegistry _registry;
@@ -124,11 +133,28 @@ internal sealed class EventPump : IAsyncDisposable
             // value consistent).
             FullMode = BoundedChannelFullMode.Wait,
             SingleReader = true,
-            SingleWriter = true,
+            // Not single-writer: besides the stream read loop, a registration that promotes
+            // parked early events (#859) writes the WakeSentinel from the subscribing thread.
+            SingleWriter = false,
         });
         _clientTag = new KeyValuePair<string, object?>("galaxy.client", clientName ?? "<unknown>");
+        _registry.ParkedEventsReady += WakeForReadyEvents;
     }
 
+    /// <summary>
+    ///     Marker written into the channel to wake the dispatch loop after a registration promoted
+    ///     parked early events (#859). Never dispatched itself; compared by reference.
+    /// </summary>
+    private static readonly MxEvent WakeSentinel = new();
+
+    /// <summary>
+    ///     Wakes the dispatch loop so promoted early events replay even if no further event arrives.
+    ///     A failed write is harmless: the channel is either full (the loop is busy and replays the
+    ///     ready queue before its next data-change dispatch) or completed (the pump is dead, and the
+    ///     next pump on this registry replays them).
+    /// </summary>
+    private void WakeForReadyEvents() => _channel.Writer.TryWrite(WakeSentinel);
+
     /// <summary>
     ///     Start consuming the event stream on a background task. Idempotent — second
     ///     calls are no-ops while the loop is running.
@@ -281,6 +307,11 @@ internal sealed class EventPump : IAsyncDisposable
         {
             await foreach (var ev in _channel.Reader.ReadAllAsync(ct).ConfigureAwait(false))
             {
+                if (ReferenceEquals(ev, WakeSentinel))
+                {
+                    DispatchReadyEvents();
+                    continue;
+                }
                 Dispatch(ev);
                 EventsDispatched.Add(1, _clientTag);
             }
@@ -313,16 +344,61 @@ internal sealed class EventPump : IAsyncDisposable
                 // — alarms come from the gateway's session-less StreamAlarms feed
                 // (GatewayGalaxyAlarmFeed). OnWriteComplete / OperationComplete /
                 // OnBufferedDataChange are filtered out: write callers get their reply
-                // via the InvokeAsync round-trip, not via the event stream.
+                // via the InvokeAsync round-trip, not via the event stream. Still replay any
+                // promoted early events: a wake lost to a full channel relies on the next item.
+                DispatchReadyEvents();
                 return;
         }
     }
 
     private void DispatchDataChange(MxEvent ev)
     {
-        var subscribers = _registry.ResolveSubscribers(ev.ItemHandle);
-        if (subscribers.Count == 0) return; // stale event after unsubscribe — drop quietly
+        var outcome = _registry.ResolveOrPark(ev, out var subscribers);
 
+        // Replay promoted early events (#859) AFTER the resolve and BEFORE fanning this event out:
+        // any registration that let this event resolve has already moved its parked (older) events
+        // to the ready queue, so they reach the handler first and per-handle order is preserved.
+        DispatchReadyEvents();
+
+        switch (outcome)
+        {
+            case EarlyEventOutcome.Resolved:
+                break;
+            case EarlyEventOutcome.Parked:
+                // The handle's SubscribeBulk reply has not been registered yet — MXAccess's initial
+                // value can overtake it. Replayed when the registration lands.
+                EarlyEventsParked.Add(1, _clientTag);
+                return;
+            case EarlyEventOutcome.Overflow:
+                EarlyEventsOverflowed.Add(1, _clientTag);
+                return;
+            default:
+                return; // stale event after unsubscribe (no subscribe in flight) — drop quietly
+        }
+
+        FanOut(ev, subscribers);
+    }
+
+    /// <summary>
+    ///     Dispatch every early event a registration promoted (#859). Runs only on the dispatch
+    ///     thread, so it is serialised with live dispatch. A promoted event whose subscription was
+    ///     removed again before the replay resolves to nobody and is dropped like any stale event.
+    /// </summary>
+    private void DispatchReadyEvents()
+    {
+        var ready = _registry.TakeReadyEvents();
+        if (ready.Count == 0) return;
+        foreach (var early in ready)
+        {
+            var subscribers = _registry.ResolveSubscribers(early.ItemHandle);
+            if (subscribers.Count == 0) continue;
+            EarlyEventsReplayed.Add(1, _clientTag);
+            FanOut(early, subscribers);
+        }
+    }
+
+    private void FanOut(MxEvent ev, IReadOnlyList<(long SubscriptionId, string FullReference)> subscribers)
+    {
         var snapshot = ToSnapshot(ev);
         foreach (var (subscriptionId, fullReference) in subscribers)
         {
@@ -377,6 +453,7 @@ internal sealed class EventPump : IAsyncDisposable
     {
         if (_disposed) return;
         _disposed = true;
+        _registry.ParkedEventsReady -= WakeForReadyEvents;
         _cts.Cancel();
         _channel.Writer.TryComplete();
         if (_loop is not null)

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/SubscriptionRegistry.cs

@@ -1,5 +1,6 @@
 using System.Collections.Concurrent;
 using System.Collections.Immutable;
+using ZB.MOM.WW.MxGateway.Contracts.Proto;
 
 namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
 
@@ -79,6 +80,7 @@ internal sealed class SubscriptionRegistry
                 _ => [subscriptionId],
                 (_, set) => set.Add(subscriptionId));
         }
+        PromoteParkedEvents(bindings);
     }
 
     /// <summary>
@@ -138,6 +140,168 @@ internal sealed class SubscriptionRegistry
     {
         _bySubscriptionId.Clear();
         _subscribersByItemHandle.Clear();
+        lock (_parkGate)
+        {
+            _parkedByItemHandle.Clear();
+            _parkedCount = 0;
+            _ready.Clear();
+        }
+    }
+
+    // ===== Early-event window (#859) =====
+    //
+    // MXAccess delivers an initial value on Advise, and nothing orders that OnDataChange behind the
+    // SubscribeBulk reply the driver registers the new item handles from:
+    //   * mxaccessgw's worker completes the STA command through a TaskCompletionSource created with
+    //     RunContinuationsAsynchronously, so the reply is framed by a thread-pool continuation while
+    //     the STA loop is already pumping the OnDataChange that Advise queued; the event queue is
+    //     drained by a separate loop that frames events as soon as it is signalled
+    //     (mxaccessgw Worker/Sta/StaCommandDispatcher.cs, Worker/Ipc/WorkerPipeSession.cs
+    //     RunEventDrainLoopAsync). The writer's control-before-event priority only orders frames
+    //     that are queued at the same time.
+    //   * the reply travels back on the unary Invoke RPC and the event on the StreamEvents RPC —
+    //     separate HTTP/2 streams with no cross-stream ordering at the client;
+    //   * and even after the reply lands, the pump's dispatch loop runs concurrently with the code
+    //     that turns the reply into bindings.
+    // So an event whose handle is not yet registered is NOT necessarily "stale after unsubscribe".
+    // While at least one subscribe is in flight such events are parked here (bounded) instead of
+    // dropped; a registration that binds a parked handle promotes its events to a ready queue that
+    // the EventPump's dispatch loop replays — on that same single thread, BEFORE any later event for
+    // the handle — so per-handle value order is preserved. When the last in-flight subscribe ends,
+    // whatever is still parked belonged to no subscription and is discarded, so the genuine
+    // stale-after-unsubscribe drop is unchanged outside the window.
+
+    /// <summary>
+    ///     Upper bound on events parked across all in-flight subscribes. Sized to hold the initial
+    ///     values of a large bulk subscribe (MXAccess sends one per advised item); overflow is
+    ///     dropped and counted by the pump, i.e. it degrades to the pre-#859 behaviour.
+    /// </summary>
+    public const int MaxParkedEvents = 100_000;
+
+    private readonly Lock _parkGate = new();
+    private readonly Dictionary<int, List<MxEvent>> _parkedByItemHandle = new();
+    private readonly Queue<MxEvent> _ready = new();
+    private int _parkedCount;
+    private int _pendingSubscribes;
+
+    /// <summary>
+    ///     Raised (outside any lock) when a registration promoted parked events to the ready queue,
+    ///     so the <see cref="EventPump"/> can wake its dispatch loop even if no further event arrives.
+    /// </summary>
+    public event Action? ParkedEventsReady;
+
+    /// <summary>Gets the number of subscribes currently inside their reply window.</summary>
+    public int PendingSubscribeCount => Volatile.Read(ref _pendingSubscribes);
+
+    /// <summary>Gets the number of events currently parked for not-yet-registered item handles.</summary>
+    public int ParkedEventCount
+    {
+        get { lock (_parkGate) return _parkedCount; }
+    }
+
+    /// <summary>
+    ///     Opens the early-event window for one subscribe. Call BEFORE issuing <c>SubscribeBulk</c>
+    ///     and dispose AFTER the returned handles are registered (or the attempt was abandoned):
+    ///     while any window is open, events for unknown handles are parked rather than dropped.
+    /// </summary>
+    /// <returns>A token that closes the window when disposed (idempotent).</returns>
+    public IDisposable BeginPendingSubscribe()
+    {
+        Interlocked.Increment(ref _pendingSubscribes);
+        return new PendingSubscribeScope(this);
+    }
+
+    private void EndPendingSubscribe()
+    {
+        if (Interlocked.Decrement(ref _pendingSubscribes) != 0) return;
+        lock (_parkGate)
+        {
+            // Re-checked under the gate: a subscribe that began after the decrement may already
+            // have parked its own early events, which must survive.
+            if (Volatile.Read(ref _pendingSubscribes) != 0) return;
+            _parkedByItemHandle.Clear();
+            _parkedCount = 0;
+        }
+    }
+
+    /// <summary>
+    ///     Resolve the subscribers for <paramref name="ev"/>'s item handle, or — when the handle is
+    ///     unknown while a subscribe is in flight — park the event for replay. Atomic with the
+    ///     promotion a registration performs, so an event is never parked after its handle's
+    ///     promotion already ran (it would otherwise be stranded until the window closed).
+    /// </summary>
+    /// <param name="ev">The data-change event.</param>
+    /// <param name="subscribers">The subscribers when the outcome is <see cref="EarlyEventOutcome.Resolved"/>.</param>
+    /// <returns>What happened to the event.</returns>
+    public EarlyEventOutcome ResolveOrPark(
+        MxEvent ev, out IReadOnlyList<(long SubscriptionId, string FullReference)> subscribers)
+    {
+        subscribers = ResolveSubscribers(ev.ItemHandle);
+        if (subscribers.Count > 0) return EarlyEventOutcome.Resolved;
+        if (Volatile.Read(ref _pendingSubscribes) == 0) return EarlyEventOutcome.Stale;
+
+        lock (_parkGate)
+        {
+            // A registration may have bound the handle between the lookup above and the gate.
+            subscribers = ResolveSubscribers(ev.ItemHandle);
+            if (subscribers.Count > 0) return EarlyEventOutcome.Resolved;
+            if (Volatile.Read(ref _pendingSubscribes) == 0) return EarlyEventOutcome.Stale;
+            if (_parkedCount >= MaxParkedEvents) return EarlyEventOutcome.Overflow;
+
+            if (!_parkedByItemHandle.TryGetValue(ev.ItemHandle, out var list))
+            {
+                list = [];
+                _parkedByItemHandle[ev.ItemHandle] = list;
+            }
+            list.Add(ev);
+            _parkedCount++;
+            return EarlyEventOutcome.Parked;
+        }
+    }
+
+    /// <summary>
+    ///     Dequeue every event a registration promoted, in arrival order. Called only from the
+    ///     EventPump's single dispatch thread.
+    /// </summary>
+    /// <returns>The promoted events; empty when none are waiting.</returns>
+    public IReadOnlyList<MxEvent> TakeReadyEvents()
+    {
+        lock (_parkGate)
+        {
+            if (_ready.Count == 0) return [];
+            var taken = _ready.ToArray();
+            _ready.Clear();
+            return taken;
+        }
+    }
+
+    private void PromoteParkedEvents(IReadOnlyList<TagBinding> bindings)
+    {
+        var promoted = false;
+        lock (_parkGate)
+        {
+            if (_parkedCount == 0) return;
+            foreach (var binding in bindings)
+            {
+                if (binding.ItemHandle <= 0) continue;
+                if (!_parkedByItemHandle.Remove(binding.ItemHandle, out var list)) continue;
+                foreach (var ev in list) _ready.Enqueue(ev);
+                _parkedCount -= list.Count;
+                promoted = true;
+            }
+        }
+        if (promoted) ParkedEventsReady?.Invoke();
+    }
+
+    private sealed class PendingSubscribeScope(SubscriptionRegistry owner) : IDisposable
+    {
+        private int _disposed;
+
+        /// <summary>Closes this subscribe's early-event window (once).</summary>
+        public void Dispose()
+        {
+            if (Interlocked.Exchange(ref _disposed, 1) == 0) owner.EndPendingSubscribe();
+        }
     }
 
     /// <summary>Snapshot every active binding for diagnostic output.</summary>
@@ -189,6 +353,7 @@ internal sealed class SubscriptionRegistry
                 _ => [subscriptionId],
                 (_, set) => set.Add(subscriptionId));
         }
+        PromoteParkedEvents(newBindings);
     }
 
     /// <summary>
@@ -227,6 +392,22 @@ internal sealed class SubscriptionRegistry
     }
 }
 
+/// <summary>Outcome of <see cref="SubscriptionRegistry.ResolveOrPark"/>.</summary>
+internal enum EarlyEventOutcome
+{
+    /// <summary>The handle is registered; dispatch to the returned subscribers.</summary>
+    Resolved,
+
+    /// <summary>No subscribe in flight — a genuinely stale event (after unsubscribe); drop it.</summary>
+    Stale,
+
+    /// <summary>Parked until the in-flight subscribe registers the handle.</summary>
+    Parked,
+
+    /// <summary>Would have been parked but <see cref="SubscriptionRegistry.MaxParkedEvents"/> was reached; dropped.</summary>
+    Overflow,
+}
+
 /// <summary>
 ///     One (full reference, gw item handle) pair returned by SubscribeBulk. Item handle is
 ///     zero or negative when the gateway rejected this individual tag (bad name, duplicate);