dohertj2/lmxopcua
1
0
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(galaxy): #865 review round 3 - budget-safe stream wait, dead pump, fast-path drop

Browse Source 
- StreamRequestWait 1 s (under Tier-A Read 2 s / Subscribe 5 s budgets) and
  a stale pump is only waited on while the supervisor is reopening or
  replaying (or a restart is underway); in a reconnect backoff the advise
  proceeds at once. The no-stream log is a rate-limited Warning while
  Healthy, Debug otherwise.
- A current-session pump whose loop has ended is not a stream: if the
  supervisor already reports Healthy (its fault was swallowed while it was
  recovering) the failure is re-reported to start a new cycle.
- Subscribe/Read no longer start the pump directly (only
  AwaitCurrentEventStreamAsync does, honouring the replay restart flag);
  the teardown refusal is an explicit ThrowIfRuntimeStopping.
- SubscriptionRegistry.ResolveOrPark fast path: the handle lookup and the
  open-window check were not atomic, so a subscribe that registered the
  handle and closed its window in between got its initial value dropped as
  stale (found by a stress run: the first event of a bulk advise lost). It
  re-looks-up before declaring stale. Deterministic test via a seam.
- Tests: first Read on a fresh driver, replay of a data subscription.

Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
Joseph Doherty
2026-11-11 13:00:19 -05:00
parent 9060ff5561
commit 1c1705fa77
4 changed files with 189 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs
+101 -19
View File
@@ -1703,7 +1703,7 @@ public sealed class GalaxyDriver
var readSubscriber = _subscriber!;
// A live pump must exist (values arrive through it); the read listens on ReadDataChange, not
// on this instance, so it survives a reconnect's pump restart.
EnsureEventPumpStarted();
ThrowIfRuntimeStopping(); // the pump itself is started by AwaitCurrentEventStreamAsync below
// #865: a read waits ONLY for the initial value, which reaches only a current-session stream that is
// already requested. The internal deadline bounds this too; a deadline (not caller) cancel falls
// through so the read still answers per-ref, as it does for a deadline during the RPC.
@@ -1960,7 +1960,7 @@ public sealed class GalaxyDriver
// generation, so the Register below can tell its SubscribeBulk straddled a teardown.
var generation = Volatile.Read(ref _runtimeGeneration);
var subscriber = _subscriber;
var pump = EnsureEventPumpStarted();
ThrowIfRuntimeStopping(); // the pump is started per attempt by AwaitCurrentEventStreamAsync (#865)
var subscriptionId = _subscriptions.NextSubscriptionId();
if (fullReferences.Count == 0)
@@ -2014,7 +2014,6 @@ public sealed class GalaxyDriver
switch (TryRegisterForGeneration(subscriptionId, bindings, generation, session))
{
case RegisterOutcome.Registered:
_ = pump; // keep the pump alive for the subscription's lifetime
return new GalaxySubscriptionHandle(subscriptionId);
case RegisterOutcome.TornDown:
@@ -2113,11 +2112,25 @@ public sealed class GalaxyDriver
}
}
/// <summary>A Subscribe/Read is refused while a teardown is in progress (the pump would belong to a dying subscriber).</summary>
private void ThrowIfRuntimeStopping()
{
lock (_pumpLock)
{
if (_runtimeStopping)
{
throw new InvalidOperationException(
"GalaxyDriver runtime is shutting down — no subscription can be served until the next InitializeAsync.");
}
}
}
/// <summary>
/// Lazily start the <see cref="EventPump"/> on the first subscribe. The pump is
/// shared across every subscription on this driver — fan-out happens through the
/// <see cref="SubscriptionRegistry"/> reverse map, not by spinning a pump per
/// subscription.
/// Lazily start the <see cref="EventPump"/> on the first advise. Since #865 the advise paths
/// (probe seed, Subscribe, Read) go through <see cref="AwaitCurrentEventStreamAsync"/>, and only the
/// replay replaces an existing pump. The pump is shared across every subscription on this driver —
/// fan-out happens through the <see cref="SubscriptionRegistry"/> reverse map, not by spinning a pump
/// per subscription.
/// </summary>
private EventPump EnsureEventPumpStarted()
{
@@ -2168,8 +2181,12 @@ public sealed class GalaxyDriver
}
}
/// <summary>Upper bound on how long an advise waits for a current-session event stream (#865).</summary>
internal TimeSpan StreamRequestWait { get; set; } = TimeSpan.FromSeconds(5);
/// <summary>
/// Upper bound on how long an advise waits for a current-session event stream (#865). Kept well under
/// Galaxy's Tier-A Read (2 s) / Subscribe (5 s) call budgets, so the "proceed anyway" fallback is
/// reachable before the resilience pipeline times the call out.
/// </summary>
internal TimeSpan StreamRequestWait { get; set; } = TimeSpan.FromSeconds(1);
/// <summary>
/// #865 — make sure an advise about to be issued has an event stream for its initial value, and return
@@ -2180,20 +2197,30 @@ public sealed class GalaxyDriver
/// A fresh StreamEvents (after_worker_sequence = 0) is never replayed, so an initial value pushed before
/// the stream attaches is lost — and a slow-changing attribute (ScanState, a setpoint) may not publish
/// again for a long time. So: start the pump if there is none, and wait until its stream call has started
/// (<see cref="EventPump.StreamRequested"/> — this narrows, it cannot close, the attach race).
/// (<see cref="EventPump.StreamRequested"/> — this narrows, it cannot close, the attach race: the gateway
/// offers no attach acknowledgement).
/// </para>
/// <para>
/// A pump bound to an OLDER session belongs to a gateway session a reconnect replaced (the reopen →
/// replay window): an advise now would go to the recreated session with no stream for it, and —
/// registered under the current session — the replay would not re-advise it. The pump is not replaced
/// here (only <see cref="ReplayAsync"/> restarts it, so two streams never race for the single-subscriber
/// slot); the advise WAITS for the replay's pump instead. The returned session is the one the pump was
/// verified against: a reconnect after this check makes the caller's registration fail as a session
/// straddle (#861), whose retry comes back through here.
/// here (only <see cref="ReplayAsync"/> restarts it, so two streams never race for the gateway's
/// single-subscriber slot); while the supervisor is actually reopening/replaying the advise WAITS for the
/// replay's pump. During a reconnect BACKOFF (TransportLost) no replay is imminent, so it does not wait.
/// The returned session is the one the pump was verified against: a reconnect after this check makes the
/// caller's registration fail as a session straddle (#861), whose retry comes back through here.
/// </para>
/// <para>
/// Bounded by <see cref="StreamRequestWait"/>; on timeout the caller proceeds with the current session
/// (logged) — no worse than the pre-#865 behaviour.
/// Bounded by <see cref="StreamRequestWait"/> per call (a #861 retry starts a new budget). On timeout the
/// caller proceeds with the current session — no worse than the pre-#865 behaviour, but note the cost:
/// such an advise registers as current, so the replay will not re-advise it, and its initial value is
/// lost until the attribute next changes.
/// </para>
/// <para>
/// A current-session pump whose loop has ENDED is not a stream (its <see cref="EventPump.StreamRequested"/>
/// completes on exit). If the supervisor already reports Healthy, its fault was swallowed while it was
/// recovering (it ignores faults outside Healthy) — the driver would be deaf while Healthy — so it is
/// reported again to start a new recovery cycle.
/// </para>
/// </remarks>
/// <param name="cancellationToken">Cancels the wait.</param>
@@ -2216,16 +2243,22 @@ public sealed class GalaxyDriver
var remaining = StreamRequestWait - elapsed.Elapsed;
if (pump is not null && pump.Session == session)
{
if (pump.HasEnded)
{
ReportEndedPump(pump);
return session;
}
if (remaining > TimeSpan.Zero)
{
try
{
await pump.StreamRequested.WaitAsync(remaining, cancellationToken).ConfigureAwait(false);
if (pump.HasEnded) ReportEndedPump(pump);
return session;
}
catch (TimeoutException)
{
// fall through to the warning
// fall through to the log
}
}
else if (pump.StreamRequested.IsCompleted)
@@ -2233,18 +2266,67 @@ public sealed class GalaxyDriver
return session;
}
}
else if (remaining > TimeSpan.Zero)
else if (remaining > TimeSpan.Zero && ReplayExpectedSoon())
{
// Reopen → replay window: the replay restarts the pump for the current session.
await Task.Delay(TimeSpan.FromMilliseconds(20), cancellationToken).ConfigureAwait(false);
continue;
}
LogNoCurrentStream();
return _subscriptions.CurrentSession;
}
}
/// <summary>
/// True when a stale pump will be replaced shortly: the supervisor is reopening/replaying, or there is
/// no supervisor (the injected-seam path, where the replay is driven directly), or a restart is underway.
/// </summary>
private bool ReplayExpectedSoon()
{
if (Volatile.Read(ref _pumpRestarting)) return true;
var supervisor = _supervisor;
if (supervisor is null) return true;
var state = supervisor.CurrentState;
return state is ReconnectSupervisor.State.Reopening or ReconnectSupervisor.State.Replaying;
}
/// <summary>A current-session pump's loop ended while the supervisor was not listening (see AwaitCurrentEventStreamAsync).</summary>
private void ReportEndedPump(EventPump pump)
{
var supervisor = _supervisor;
if (supervisor is null || _runtimeStopping || supervisor.CurrentState != ReconnectSupervisor.State.Healthy) return;
lock (_pumpLock)
{
if (!ReferenceEquals(_eventPump, pump)) return; // already replaced
}
_logger.LogWarning(
"GalaxyDriver {InstanceId} event stream has ended while the reconnect supervisor reports Healthy — re-reporting the transport failure.",
_driverInstanceId);
try { supervisor.ReportTransportFailure(new InvalidOperationException("The Galaxy event stream ended; no events can be received.")); }
catch (ObjectDisposedException) { /* driver disposing */ }
}
private long _lastNoStreamWarningTicks;
/// <summary>Warning at most once a minute while the supervisor is Healthy; Debug otherwise (an outage would flood).</summary>
private void LogNoCurrentStream()
{
var supervisor = _supervisor;
var now = Environment.TickCount64;
var last = Interlocked.Read(ref _lastNoStreamWarningTicks);
if ((supervisor is null || supervisor.CurrentState == ReconnectSupervisor.State.Healthy)
&& (last == 0 || now - last >= 60_000)
&& Interlocked.CompareExchange(ref _lastNoStreamWarningTicks, now, last) == last)
{
_logger.LogWarning(
"GalaxyDriver {InstanceId} had no started event stream for the current gateway session within {Wait} — advising anyway; initial values may be missed.",
_driverInstanceId, StreamRequestWait);
return _subscriptions.CurrentSession;
return;
}
_logger.LogDebug(
"GalaxyDriver {InstanceId} advising without a started current-session event stream (supervisor not Healthy or warning rate-limited).",
_driverInstanceId);
}
/// <summary>
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/SubscriptionRegistry.cs
+18 -1
View File
@@ -314,6 +314,12 @@ internal sealed class SubscriptionRegistry
/// </summary>
internal Action? AfterMapWriteForTests { get; set; }
/// <summary>
/// Test seam (#865): invoked on <see cref="ResolveOrPark"/>'s lock-free fast path after the handle
/// lookup missed, before the open-window check.
/// </summary>
internal Action? AfterFastPathMissForTests { get; set; }
/// <summary>
/// Test seam (M-1): parks <paramref name="ev"/> unconditionally, producing the parked list for a
/// resolved handle that the atomic registration otherwise makes impossible.
@@ -496,7 +502,18 @@ internal sealed class SubscriptionRegistry
{
subscribers = ResolveSubscribers(ev.ItemHandle);
if (subscribers.Count > 0) return EarlyEventOutcome.Resolved;
if (Volatile.Read(ref _pendingSubscribes) == 0) return EarlyEventOutcome.Stale;
AfterFastPathMissForTests?.Invoke();
if (Volatile.Read(ref _pendingSubscribes) == 0)
{
// #865 (found by its stress run): the lookup above and this read are not atomic. A
// subscribe can register the handle AND close its window in between — then "no window
// open" does not mean "stale", and dropping here lost the initial value for good (the
// first event of a bulk advise, while the dispatch thread was slow). The window closes
// only after its registration, so once no window is open a re-lookup sees every
// registration that window made; nothing is held, so no older event can be overtaken.
subscribers = ResolveSubscribers(ev.ItemHandle);
return subscribers.Count > 0 ? EarlyEventOutcome.Resolved : EarlyEventOutcome.Stale;
}
}
lock (_parkGate)