dohertj2/lmxopcua
1
0
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects Releases Wiki Activity

Merge #864/#865 Galaxy host status: per-template platform seed, event stream started before probe registration; registry and reconnect races fixed

Browse Source 
Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
Joseph Doherty
2026-11-11 13:22:33 -05:00
parent 50b7b030a9 59d33ab938
commit b5d4e25905
12 changed files with 1197 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Browse/GatewayGalaxyPlatformSource.cs
+41 -28
View File
@@ -5,11 +5,19 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
/// <summary>
/// Default <see cref="IGalaxyPlatformSource"/>: a platform-only <c>DiscoverHierarchy</c> sweep
/// over the gateway's <see cref="GalaxyRepositoryClient"/> — the request filters on
/// <c>template_chain_contains</c> (<c>$WinPlatform</c>, <c>$AppEngine</c>) and asks for no
/// attributes, so it is a few rows per galaxy rather than the full object + attribute dump the
/// address-space discovery pulls. Paged at the configured <c>Repository.DiscoverPageSize</c>.
/// over the gateway's <see cref="GalaxyRepositoryClient"/> — one sweep per platform template
/// (<c>$WinPlatform</c>, then <c>$AppEngine</c>), each filtered on that single
/// <c>template_chain_contains</c> token with no attributes, so it is a few rows per galaxy rather
/// than the full object + attribute dump the address-space discovery pulls. Paged at the
/// configured <c>Repository.DiscoverPageSize</c>.
/// </summary>
/// <remarks>
/// ⚠️ <b>One template per request (#864).</b> The gateway's shared
/// <c>GalaxyHierarchyProjector.MatchesFilters</c> ANDs the <c>template_chain_contains</c> entries —
/// an object must match <i>every</i> token — so a single request carrying both templates matches no
/// platform (a <c>$WinPlatform</c>'s chain has no <c>$AppEngine</c>, and vice versa) and the probe
/// watcher seeded 0 platforms in production. The sweeps are unioned and de-duplicated by tag name.
/// </remarks>
public sealed class GatewayGalaxyPlatformSource : IGalaxyPlatformSource
{
/// <summary>Template-chain tokens that identify a platform / engine object.</summary>
@@ -32,15 +40,16 @@ public sealed class GatewayGalaxyPlatformSource : IGalaxyPlatformSource
=> QueryAsync(_client.DiscoverHierarchyRawAsync, _pageSize, cancellationToken);
/// <summary>
/// The platform-only sweep behind a delegate so it is unit-testable without a channel. Every
/// request carries the template filter and <c>include_attributes = false</c>; the reply is
/// The platform-only sweeps behind a delegate so it is unit-testable without a channel. One
/// paged sweep per entry of <see cref="PlatformTemplates"/>; every request carries exactly that
/// one template token and <c>include_attributes = false</c>; the reply is
/// re-filtered on the returned template chain (a gateway that ignores the filter still
/// yields only platforms) and reduced to non-empty tag names.
/// </summary>
/// <param name="discover">The raw <c>DiscoverHierarchy</c> RPC.</param>
/// <param name="pageSize">Objects per page.</param>
/// <param name="cancellationToken">Cancellation token for the walk.</param>
/// <returns>Distinct platform tag names in gateway order.</returns>
/// <returns>Distinct platform tag names: <c>$WinPlatform</c> sweep first, then <c>$AppEngine</c>, each in gateway order.</returns>
internal static async Task<IReadOnlyList<string>> QueryAsync(
Func<DiscoverHierarchyRequest, CancellationToken, Task<DiscoverHierarchyReply>> discover,
int pageSize,
@@ -51,33 +60,37 @@ public sealed class GatewayGalaxyPlatformSource : IGalaxyPlatformSource
var names = new List<string>();
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var seenPageTokens = new HashSet<string>(StringComparer.Ordinal);
var pageToken = string.Empty;
do
foreach (var template in PlatformTemplates)
{
var request = new DiscoverHierarchyRequest
var seenPageTokens = new HashSet<string>(StringComparer.Ordinal);
var pageToken = string.Empty;
do
{
PageSize = effective,
PageToken = pageToken,
IncludeAttributes = false,
};
request.TemplateChainContains.Add(PlatformTemplates);
var request = new DiscoverHierarchyRequest
{
PageSize = effective,
PageToken = pageToken,
IncludeAttributes = false,
};
// Exactly ONE token per request: the gateway ANDs template filters (#864).
request.TemplateChainContains.Add(template);
var reply = await discover(request, cancellationToken).ConfigureAwait(false);
foreach (var obj in reply.Objects)
{
if (string.IsNullOrEmpty(obj.TagName) || !IsPlatform(obj)) continue;
if (seen.Add(obj.TagName)) names.Add(obj.TagName);
}
var reply = await discover(request, cancellationToken).ConfigureAwait(false);
foreach (var obj in reply.Objects)
{
if (string.IsNullOrEmpty(obj.TagName) || !IsPlatform(obj)) continue;
if (seen.Add(obj.TagName)) names.Add(obj.TagName);
}
pageToken = reply.NextPageToken ?? string.Empty;
if (!string.IsNullOrWhiteSpace(pageToken) && !seenPageTokens.Add(pageToken))
{
throw new InvalidOperationException(
$"Galaxy DiscoverHierarchy (platform query) returned a repeated page token '{pageToken}' — refusing to loop.");
pageToken = reply.NextPageToken ?? string.Empty;
if (!string.IsNullOrWhiteSpace(pageToken) && !seenPageTokens.Add(pageToken))
{
throw new InvalidOperationException(
$"Galaxy DiscoverHierarchy (platform query, {template}) returned a repeated page token '{pageToken}' — refusing to loop.");
}
}
while (!string.IsNullOrWhiteSpace(pageToken));
}
while (!string.IsNullOrWhiteSpace(pageToken));
return names;
}
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/GalaxyDriver.cs
+310 -27
View File
@@ -540,6 +540,9 @@ public sealed class GalaxyDriver
// a cold gateway cache's first-load wait — longer than Galaxy's Tier-A Initialize budget. It
// runs in the background: InitializeAsync neither waits on it nor fails because of it.
await PrepareProbeSeedAsync(cancellationToken).ConfigureAwait(false);
// #865: the seed's probe advises start the event pump themselves (BuildProbeWatcher's
// pre-subscribe hook) — a driver with no authored tags never Reads/Subscribes, and the pump
// used to start only there, so every initial ScanState value went to no stream.
RequestProbeSeed("initialize");
_logger.LogInformation(
"GalaxyDriver {InstanceId} initialized — endpoint={Endpoint} clientName={ClientName}",
@@ -730,9 +733,12 @@ public sealed class GalaxyDriver
// The watcher publishes its probe handles into the driver's registry (reserved id) so the
// EventPump's fan-out reaches OnPumpDataChange for ScanState events — without that every
// probe event was dropped before it got anywhere near the watcher.
// #865: the watcher also starts the pump before every probe SubscribeBulk — the initial ScanState
// value only reaches a stream that is already attached (the gateway does not replay a fresh one).
private PerPlatformProbeWatcher BuildProbeWatcher(IGalaxySubscriber subscriber) =>
new(subscriber, _hostStatuses, _subscriptions, _logger,
bufferedUpdateIntervalMs: _options.MxAccess.PublishingIntervalMs);
bufferedUpdateIntervalMs: _options.MxAccess.PublishingIntervalMs,
ensureEventStream: EnsureProbeEventStreamAsync);
/// <summary>
/// Build the production gw client + session + per-capability runtime components
@@ -1047,6 +1053,26 @@ public sealed class GalaxyDriver
// subscriptions to replay the dead pump used to stay in _eventPump, so every later
// Subscribe/Read reused it and was silently deaf while health stayed Healthy.
RestartEventPumpForReplay();
// #865: every re-advise below pushes its initial value once, to whatever stream is attached.
EventPump? replayPump;
lock (_pumpLock) replayPump = _eventPump;
if (replayPump is not null)
{
await AwaitCurrentEventStreamAsync(cancellationToken).ConfigureAwait(false);
// A replay whose fresh pump already died must FAIL, so the supervisor backs off and stays
// Degraded — succeeding would go Healthy with a dead stream, and the next advise's re-report
// would start a new cycle at attempt 1 (no backoff): a tight reconnect loop.
lock (_pumpLock) replayPump = _eventPump;
if (replayPump is { HasEnded: true })
{
// A teardown cancelling this replay can end the pump too: that is a clean cancellation.
cancellationToken.ThrowIfCancellationRequested();
// The synchronous case; a stream that ends a round trip later reports a transport failure
// while Replaying, which the supervisor itself turns into a failed attempt.
throw new InvalidOperationException(
"The Galaxy event stream ended during the reconnect replay — retrying after backoff.");
}
}
// Only subscriptions still holding the previous session's handles: one registered after the
// reopen is already live on the recreated session (re-subscribing it would double-advise).
@@ -1302,27 +1328,45 @@ public sealed class GalaxyDriver
{
old = _eventPump;
if (old is null) return; // pump never started — nothing to restart
// #865: a pump first started after the reopen's fence (no pump existed before the reconnect)
// already streams the recreated session, and advises registered under it may be waiting for
// their initial values on it — keep it, unless its loop has ended (a dead pump is always replaced,
// #859 P-1: a stream fault during Replaying is not re-reported by the supervisor).
if (old.Session == _subscriptions.CurrentSession && !old.HasEnded) return;
_eventPump = null;
// #865: until the replacement exists, AwaitCurrentEventStreamAsync must not start one of its
// own — two streams would race for the gateway's single-subscriber slot while the old one detaches.
_pumpRestarting = true;
}
// Detach + dispose the faulted pump outside the lock so a slow shutdown doesn't
// block a concurrent EnsureEventPumpStarted; the old loop already exited on fault.
old.OnDataChange -= OnPumpDataChange;
try { old.DisposeAsync().AsTask().GetAwaiter().GetResult(); }
catch (Exception ex)
try
{
_logger.LogWarning(ex,
"GalaxyDriver {InstanceId} faulted EventPump dispose failed during replay — continuing.",
_driverInstanceId);
}
// Detach + dispose the faulted pump outside the lock so a slow shutdown doesn't
// block a concurrent EnsureEventPumpStarted; the old loop already exited on fault.
old.OnDataChange -= OnPumpDataChange;
try { old.DisposeAsync().AsTask().GetAwaiter().GetResult(); }
catch (Exception ex)
{
_logger.LogWarning(ex,
"GalaxyDriver {InstanceId} faulted EventPump dispose failed during replay — continuing.",
_driverInstanceId);
}
// EnsureEventPumpStarted creates + starts a fresh pump under the lock. Skip the
// recreate if the driver is being disposed or its runtime torn down — the teardown
// already released the pump and a fresh one here would leak past it.
if (_disposed || _runtimeStopping) return;
EnsureEventPumpStarted();
// EnsureEventPumpStarted creates + starts a fresh pump under the lock. Skip the
// recreate if the driver is being disposed or its runtime torn down — the teardown
// already released the pump and a fresh one here would leak past it.
if (_disposed || _runtimeStopping) return;
EnsureEventPumpStarted();
}
finally
{
lock (_pumpLock) _pumpRestarting = false;
}
}
/// <summary>True while <see cref="RestartEventPumpForReplay"/> is between dropping the old pump and starting its replacement.</summary>
private bool _pumpRestarting;
private void OnSupervisorStateChanged(object? sender, StateTransition transition)
{
// Reflect supervisor state in DriverHealth + transport forwarder.
@@ -1675,7 +1719,19 @@ public sealed class GalaxyDriver
var readSubscriber = _subscriber!;
// A live pump must exist (values arrive through it); the read listens on ReadDataChange, not
// on this instance, so it survives a reconnect's pump restart.
EnsureEventPumpStarted();
ThrowIfRuntimeStopping(); // the pump itself is started by AwaitCurrentEventStreamAsync below
// #865: a read waits ONLY for the initial value, which reaches only a current-session stream that is
// already requested. The internal deadline bounds this too; a deadline (not caller) cancel falls
// through so the read still answers per-ref, as it does for a deadline during the RPC.
int issueSession;
try
{
issueSession = await AwaitCurrentEventStreamAsync(cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when (!callerToken.IsCancellationRequested)
{
issueSession = _subscriptions.CurrentSession;
}
var subscriptionId = _subscriptions.NextSubscriptionId();
// Pre-allocate one TaskCompletionSource per full-reference so the OnDataChange
@@ -1712,7 +1768,7 @@ public sealed class GalaxyDriver
// window before the RPC and close it right after registration (the finally below), not after the
// value wait: a window held for the whole wait would park every unrelated unknown-handle event.
var earlyEventWindow = _subscriptions.BeginPendingSubscribe();
var session = _subscriptions.CurrentSession; // #861 — the session this SubscribeBulk is issued on
var session = issueSession; // #861/#865 — the session this SubscribeBulk is issued on (its stream verified)
try
{
// The RPC gets only the CALLER's token: the internal deadline bounds our wait, not the RPC, so a slow
@@ -1920,7 +1976,7 @@ public sealed class GalaxyDriver
// generation, so the Register below can tell its SubscribeBulk straddled a teardown.
var generation = Volatile.Read(ref _runtimeGeneration);
var subscriber = _subscriber;
var pump = EnsureEventPumpStarted();
ThrowIfRuntimeStopping(); // the pump is started per attempt by AwaitCurrentEventStreamAsync (#865)
var subscriptionId = _subscriptions.NextSubscriptionId();
if (fullReferences.Count == 0)
@@ -1953,6 +2009,10 @@ public sealed class GalaxyDriver
for (var attempt = 1; ; attempt++)
{
// #865: per attempt (a #861 retry is a new session): the advise's initial value reaches only a
// current-session stream that is already requested — the gateway does not replay a fresh stream.
var issueSession = await AwaitCurrentEventStreamAsync(cancellationToken).ConfigureAwait(false);
// #859: MXAccess's initial value for a new handle can reach the event pump before this
// reply is registered (the gateway gives no reply-before-event ordering, and the reply and
// the event ride separate RPCs). Open the early-event window BEFORE the RPC so such events
@@ -1961,7 +2021,7 @@ public sealed class GalaxyDriver
var earlyEventWindow = _subscriptions.BeginPendingSubscribe();
try
{
var session = _subscriptions.CurrentSession; // #861 — the session this SubscribeBulk is issued on
var session = issueSession; // #861/#865 — the session this SubscribeBulk is issued on (its stream verified)
var results = await subscriber
.SubscribeBulkAsync(dialledRefs, bufferedIntervalMs, cancellationToken)
.ConfigureAwait(false);
@@ -1970,7 +2030,6 @@ public sealed class GalaxyDriver
switch (TryRegisterForGeneration(subscriptionId, bindings, generation, session))
{
case RegisterOutcome.Registered:
_ = pump; // keep the pump alive for the subscription's lifetime
return new GalaxySubscriptionHandle(subscriptionId);
case RegisterOutcome.TornDown:
@@ -2069,11 +2128,25 @@ public sealed class GalaxyDriver
}
}
/// <summary>A Subscribe/Read is refused while a teardown is in progress (the pump would belong to a dying subscriber).</summary>
private void ThrowIfRuntimeStopping()
{
lock (_pumpLock)
{
if (_runtimeStopping)
{
throw new InvalidOperationException(
"GalaxyDriver runtime is shutting down — no subscription can be served until the next InitializeAsync.");
}
}
}
/// <summary>
/// Lazily start the <see cref="EventPump"/> on the first subscribe. The pump is
/// shared across every subscription on this driver — fan-out happens through the
/// <see cref="SubscriptionRegistry"/> reverse map, not by spinning a pump per
/// subscription.
/// Lazily start the <see cref="EventPump"/> on the first advise. Since #865 the advise paths
/// (probe seed, Subscribe, Read) go through <see cref="AwaitCurrentEventStreamAsync"/>, and only the
/// replay replaces an existing pump. The pump is shared across every subscription on this driver —
/// fan-out happens through the <see cref="SubscriptionRegistry"/> reverse map, not by spinning a pump
/// per subscription.
/// </summary>
private EventPump EnsureEventPumpStarted()
{
@@ -2089,18 +2162,206 @@ public sealed class GalaxyDriver
"GalaxyDriver runtime is shutting down — no subscription can be served until the next InitializeAsync.");
}
if (_eventPump is not null) return _eventPump;
_eventPump = new EventPump(
// The fault callback carries the pump's identity, so a late fault from a pump that is no
// longer the live one (a stale pump of a reopened session) is not escalated (#865 review).
EventPump? created = null;
created = new EventPump(
_subscriber!, _subscriptions, _logger,
channelCapacity: _options.MxAccess.EventPumpChannelCapacity,
clientName: _options.MxAccess.ClientName,
onStreamFault: OnEventPumpStreamFault,
onStreamFault: cause => OnEventPumpStreamFault(created, cause),
onReplayGap: OnEventPumpReplayGap);
_eventPump = created;
_eventPump.OnDataChange += OnPumpDataChange;
_eventPump.Start();
return _eventPump;
}
}
/// <summary>
/// <see cref="EnsureEventPumpStarted"/> for infrastructure callers that must not fail because the
/// runtime is going away: null — and no pump — while a teardown is in progress or there is no subscriber.
/// </summary>
/// <returns>The running (or just started) pump; null when none may be started.</returns>
private EventPump? TryEnsureEventPumpStarted()
{
lock (_pumpLock)
{
if (_runtimeStopping || _disposed || _subscriber is null) return null;
try
{
return EnsureEventPumpStarted(); // re-enters _pumpLock (Lock is reentrant)
}
catch (InvalidOperationException)
{
// _runtimeStopping is written without _pumpLock; a teardown that flipped it between the
// check above and EnsureEventPumpStarted's own check is the same "no pump" answer.
return null;
}
}
}
/// <summary>
/// Upper bound on how long an advise waits for a current-session event stream (#865). Kept well under
/// Galaxy's Tier-A Read (2 s) / Subscribe (5 s) call budgets, so the "proceed anyway" fallback is
/// reachable before the resilience pipeline times the call out.
/// </summary>
internal TimeSpan StreamRequestWait { get; set; } = TimeSpan.FromSeconds(1);
/// <summary>
/// #865 — make sure an advise about to be issued has an event stream for its initial value, and return
/// the registry session to issue (and register) it under.
/// </summary>
/// <remarks>
/// <para>
/// A fresh StreamEvents (after_worker_sequence = 0) is never replayed, so an initial value pushed before
/// the stream attaches is lost — and a slow-changing attribute (ScanState, a setpoint) may not publish
/// again for a long time. So: start the pump if there is none, and wait until its stream call has started
/// (<see cref="EventPump.StreamRequested"/> — this narrows, it cannot close, the attach race: the gateway
/// offers no attach acknowledgement).
/// </para>
/// <para>
/// A pump bound to an OLDER session belongs to a gateway session a reconnect replaced (the reopen →
/// replay window): an advise now would go to the recreated session with no stream for it, and —
/// registered under the current session — the replay would not re-advise it. The pump is not replaced
/// here (only <see cref="ReplayAsync"/> restarts it, so two streams never race for the gateway's
/// single-subscriber slot); while the supervisor is actually reopening/replaying the advise WAITS for the
/// replay's pump. During a reconnect BACKOFF (TransportLost) no replay is imminent, so it does not wait.
/// The returned session is the one the pump was verified against: a reconnect after this check makes the
/// caller's registration fail as a session straddle (#861), whose retry comes back through here.
/// </para>
/// <para>
/// Bounded by <see cref="StreamRequestWait"/> per call (a #861 retry starts a new budget). On timeout the
/// caller proceeds with the current session — no worse than the pre-#865 behaviour, but note the cost:
/// such an advise registers as current, so the replay will not re-advise it, and its initial value is
/// lost until the attribute next changes.
/// </para>
/// <para>
/// A current-session pump whose loop has ENDED is not a stream (its <see cref="EventPump.StreamRequested"/>
/// completes on exit). If the supervisor already reports Healthy, its fault was swallowed while it was
/// recovering (it ignores faults outside Healthy) — the driver would be deaf while Healthy — so it is
/// reported again to start a new recovery cycle.
/// </para>
/// </remarks>
/// <param name="cancellationToken">Cancels the wait.</param>
/// <returns>The registry session to issue the advise under.</returns>
private async Task<int> AwaitCurrentEventStreamAsync(CancellationToken cancellationToken)
{
var elapsed = System.Diagnostics.Stopwatch.StartNew();
while (true)
{
var session = _subscriptions.CurrentSession;
EventPump? pump;
bool restarting;
lock (_pumpLock)
{
restarting = _pumpRestarting;
pump = restarting ? null : TryEnsureEventPumpStarted();
}
if (pump is null && !restarting) return session; // tearing down — the caller's own fences decide
var remaining = StreamRequestWait - elapsed.Elapsed;
if (pump is not null && pump.Session == session)
{
if (pump.HasEnded)
{
ReportEndedPump(pump);
return session;
}
if (remaining > TimeSpan.Zero)
{
try
{
await pump.StreamRequested.WaitAsync(remaining, cancellationToken).ConfigureAwait(false);
if (pump.HasEnded) ReportEndedPump(pump);
return session;
}
catch (TimeoutException)
{
// fall through to the log
}
}
else if (pump.StreamRequested.IsCompleted)
{
return session;
}
}
else if (remaining > TimeSpan.Zero && ReplayExpectedSoon())
{
// Reopen → replay window: the replay restarts the pump for the current session.
await Task.Delay(TimeSpan.FromMilliseconds(20), cancellationToken).ConfigureAwait(false);
continue;
}
LogNoCurrentStream();
return _subscriptions.CurrentSession;
}
}
/// <summary>
/// True when a stale pump will be replaced shortly: a reopen/replay attempt is running, or there is
/// no supervisor (the injected-seam path, where the replay is driven directly), or a restart is underway.
/// </summary>
private bool ReplayExpectedSoon()
{
if (Volatile.Read(ref _pumpRestarting)) return true;
var supervisor = _supervisor;
if (supervisor is null) return true;
// Not the state: between attempts (the backoff delay, up to MaxBackoff) it still reads
// Reopening/Replaying while no replay is coming any time soon.
return supervisor.AttemptInProgress;
}
/// <summary>A current-session pump's loop ended while the supervisor was not listening (see AwaitCurrentEventStreamAsync).</summary>
private void ReportEndedPump(EventPump pump)
{
var supervisor = _supervisor;
if (supervisor is null || _runtimeStopping || supervisor.CurrentState != ReconnectSupervisor.State.Healthy) return;
lock (_pumpLock)
{
if (!ReferenceEquals(_eventPump, pump)) return; // already replaced
}
_logger.LogWarning(
"GalaxyDriver {InstanceId} event stream has ended while the reconnect supervisor reports Healthy — re-reporting the transport failure.",
_driverInstanceId);
try { supervisor.ReportTransportFailure(new InvalidOperationException("The Galaxy event stream ended; no events can be received.")); }
catch (ObjectDisposedException) { /* driver disposing */ }
}
private long _lastNoStreamWarningTicks;
/// <summary>Warning at most once a minute while the supervisor is Healthy; Debug otherwise (an outage would flood).</summary>
private void LogNoCurrentStream()
{
var supervisor = _supervisor;
var now = Environment.TickCount64;
var last = Interlocked.Read(ref _lastNoStreamWarningTicks);
if ((supervisor is null || supervisor.CurrentState == ReconnectSupervisor.State.Healthy)
&& (last == 0 || now - last >= 60_000)
&& Interlocked.CompareExchange(ref _lastNoStreamWarningTicks, now, last) == last)
{
_logger.LogWarning(
"GalaxyDriver {InstanceId} had no started event stream for the current gateway session within {Wait} — advising anyway; initial values may be missed.",
_driverInstanceId, StreamRequestWait);
return;
}
_logger.LogDebug(
"GalaxyDriver {InstanceId} advising without a started current-session event stream (supervisor not Healthy or warning rate-limited).",
_driverInstanceId);
}
/// <summary>
/// #865 — the probe watcher's pre-subscribe hook (<see cref="AwaitCurrentEventStreamAsync"/>). A probe's
/// initial ScanState value is usually its only event, and a driver with no authored tags never
/// Reads/Subscribes, so without this its pump — and every host status — never existed.
/// </summary>
private Task<int> EnsureProbeEventStreamAsync(CancellationToken cancellationToken)
{
// A seed pass a teardown already cancelled must not start a pump after the teardown finished.
cancellationToken.ThrowIfCancellationRequested();
return AwaitCurrentEventStreamAsync(cancellationToken);
}
/// <summary>
/// Replay-gap callback for the <see cref="EventPump"/>. The gateway told us it evicted
/// events we can never receive, so every cached value may be stale by an unknown amount.
@@ -2184,7 +2445,12 @@ public sealed class GalaxyDriver
/// replay; without this hand-off a transient transport drop permanently kills
/// the event stream and <c>GetHealth()</c> keeps reporting Healthy.
/// </summary>
private void OnEventPumpStreamFault(Exception cause)
private int _escalatedStreamFaults;
/// <summary>Test seam (#865): stream faults that passed the superseded-pump filter (escalated toward the supervisor).</summary>
internal int EscalatedStreamFaultsForTests => Volatile.Read(ref _escalatedStreamFaults);
private void OnEventPumpStreamFault(EventPump? source, Exception cause)
{
if (_runtimeStopping)
{
@@ -2194,6 +2460,23 @@ public sealed class GalaxyDriver
return;
}
if (source is not null)
{
bool stale;
lock (_pumpLock) stale = !ReferenceEquals(_eventPump, source) || source.Session != _subscriptions.CurrentSession;
if (stale)
{
// A pump that is no longer the live one: the identity check does the work (the replay
// replaces/replaced it). The session part only matters between a reopen and its replay,
// where the supervisor ignores faults anyway — ⚠️ a future session bump while Healthy that
// keeps the same pump would make its real faults look stale here. Its fault says nothing
// about the current transport, and reported while Replaying it would fail a good replay.
_logger.LogDebug(cause, "GalaxyDriver {InstanceId} ignored a stream fault from a superseded event pump.", _driverInstanceId);
return;
}
}
Interlocked.Increment(ref _escalatedStreamFaults);
var supervisor = _supervisor;
if (supervisor is null)
{
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Health/PerPlatformProbeWatcher.cs
+23 -2
View File
@@ -44,6 +44,11 @@ public sealed class PerPlatformProbeWatcher : IDisposable
// driven directly (unit tests of the decoding table).
private readonly SubscriptionRegistry? _registry;
// #865: invoked before every probe SubscribeBulk so the driver's event stream is attached when the
// gateway pushes the initial ScanState value (a fresh StreamEvents does not replay earlier events,
// and ScanState rarely changes — a lost initial value leaves the host missing indefinitely).
private readonly Func<CancellationToken, Task<int>>? _ensureEventStream;
// Tracked platform → gw item handle. Item handle 0 means the gw rejected the subscribe;
// we keep the entry so SyncPlatformsAsync doesn't try to subscribe it again on every call.
// Each handle carries the registry session that issued it (#859 I-A): item handles are per gateway
@@ -79,16 +84,24 @@ public sealed class PerPlatformProbeWatcher : IDisposable
/// <param name="registry">The driver's subscription registry; null to keep the handles private.</param>
/// <param name="logger">Optional logger for diagnostic messages.</param>
/// <param name="bufferedUpdateIntervalMs">Buffered update interval in milliseconds; must be >= 0.</param>
/// <param name="ensureEventStream">
/// Optional: awaited before every probe SubscribeBulk to make sure the event stream that carries
/// the initial ScanState value is running and requested for the current gateway session (the
/// driver starts its <see cref="EventPump"/> and waits for its StreamEvents call). Returns the
/// registry session the stream was verified against; the advise is issued and fenced under it.
/// </param>
internal PerPlatformProbeWatcher(
IGalaxySubscriber subscriber,
HostStatusAggregator aggregator,
SubscriptionRegistry? registry,
ILogger? logger = null,
int bufferedUpdateIntervalMs = 0)
int bufferedUpdateIntervalMs = 0,
Func<CancellationToken, Task<int>>? ensureEventStream = null)
{
_subscriber = subscriber ?? throw new ArgumentNullException(nameof(subscriber));
_aggregator = aggregator ?? throw new ArgumentNullException(nameof(aggregator));
_registry = registry;
_ensureEventStream = ensureEventStream;
_logger = logger ?? NullLogger.Instance;
if (bufferedUpdateIntervalMs < 0)
{
@@ -196,10 +209,18 @@ public sealed class PerPlatformProbeWatcher : IDisposable
{
var probeAddresses = platforms.Select(p => p + ProbeSuffix).ToArray();
// #865: the stream must be live BEFORE the advise — the initial value is the probe's only
// event until the platform's scan state changes, and an unattached stream never sees it.
// The returned session is the one the stream was verified for: a reconnect after the check makes
// the session test below refuse the reply as a straddle, and the retry comes back through here.
int? streamSession = _ensureEventStream is null
? null
: await _ensureEventStream(cancellationToken).ConfigureAwait(false);
// #859: the initial ScanState value can overtake the SubscribeBulk reply; park it until
// PublishBindingsLocked registers the handles below (window closes at method exit).
using var earlyEventWindow = _registry?.BeginPendingSubscribe();
var session = CurrentSession; // the session this SubscribeBulk is issued on
var session = streamSession ?? CurrentSession; // the session this SubscribeBulk is issued on
// PR 6.3 — use the configured bufferedUpdateIntervalMs (defaults to 0 = gw cadence
// when the driver hasn't overridden MxAccess.PublishingIntervalMs). Probe ScanState
// changes are rare so a coarser interval is usually fine; deployments that need
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/EventPump.cs
+65 -1
View File
@@ -148,6 +148,36 @@ internal sealed class EventPump : IAsyncDisposable
private readonly int _session;
/// <summary>
/// The registry session this pump was created for (#865). A pump of an older session was opened
/// against a gateway session a reconnect replaced; its stream will never carry the new session's events.
/// </summary>
internal int Session => _session;
/// <summary>True once the stream loop has exited (faulted, completed by the gateway, or disposed).</summary>
internal bool HasEnded => Volatile.Read(ref _ended);
// Set at the very start of the stream loop's finally — before StreamRequested is completed there — so a
// waiter released by a dying loop always sees it ended (the loop task itself completes only later).
private bool _ended;
private readonly TaskCompletionSource _streamRequested = new(TaskCreationOptions.RunContinuationsAsynchronously);
/// <summary>
/// Completes once the stream loop has STARTED its <c>StreamEvents</c> call (the first
/// <c>MoveNextAsync</c> runs the subscriber's iterator synchronously up to the gRPC call creation) —
/// or once the loop has ended without getting that far, so a waiter never hangs on a dead pump.
/// #865: the gateway does not replay a fresh stream, so a caller whose only expected event is the
/// initial value of an advise it is about to make waits on this first.
/// </summary>
/// <remarks>
/// ⚠️ This NARROWS the stream-attach race, it does not close it: the HTTP/2 request is sent
/// asynchronously and the gateway attaches the subscriber after auth, concurrently with any other
/// RPC. In practice the attach wins (an advise's initial value needs a worker IPC round trip plus an
/// MXAccess advise), but the gateway offers no attach acknowledgement to await.
/// </remarks>
public Task StreamRequested => _streamRequested.Task;
/// <summary>
/// Marker written into the channel to wake the dispatch loop after a registration promoted
/// parked early events (#859). Never dispatched itself; compared by reference.
@@ -179,10 +209,29 @@ internal sealed class EventPump : IAsyncDisposable
private async Task RunAsync(CancellationToken ct)
{
IAsyncEnumerator<MxEvent>? enumerator = null;
try
{
await foreach (var ev in _subscriber.StreamEventsAsync(ct).WithCancellation(ct).ConfigureAwait(false))
enumerator = _subscriber.StreamEventsAsync(ct).GetAsyncEnumerator(ct);
// Start the first MoveNext BEFORE awaiting it: that synchronous step issues the RPC, which is
// the moment StreamRequested reports (#865).
var next = enumerator.MoveNextAsync();
if (next.IsCompleted)
{
// Already decided (an item, an immediate end, or a fault): consume it now, so a stream that
// ended or faulted at once is seen as ended (finally) BEFORE StreamRequested releases anyone.
var hasFirst = await next.ConfigureAwait(false);
if (!hasFirst) Volatile.Write(ref _ended, true);
_streamRequested.TrySetResult();
next = new ValueTask<bool>(hasFirst);
}
else
{
_streamRequested.TrySetResult();
}
for (; await next.ConfigureAwait(false); next = enumerator.MoveNextAsync())
{
var ev = enumerator.Current;
if (ct.IsCancellationRequested) break;
EventsReceived.Add(1, _clientTag);
@@ -241,6 +290,21 @@ internal sealed class EventPump : IAsyncDisposable
}
finally
{
Volatile.Write(ref _ended, true);
// A loop that died before issuing the RPC must not strand a StreamRequested waiter.
_streamRequested.TrySetResult();
if (enumerator is not null)
{
try { await enumerator.DisposeAsync().ConfigureAwait(false); }
catch (Exception ex) when (ct.IsCancellationRequested || ex is OperationCanceledException)
{
// Disposing a cancelled stream may surface its cancellation — shutdown noise.
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Galaxy EventPump stream enumerator dispose failed.");
}
}
// Tell the dispatch loop the producer is done so it drains and exits.
_channel.Writer.TryComplete();
}
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/ReconnectSupervisor.cs
+62 -10
View File
@@ -104,6 +104,18 @@ public sealed class ReconnectSupervisor : IDisposable
get { lock (_stateLock) return _state; }
}
private bool _attemptInProgress;
// Set (under _stateLock) by a ReportTransportFailure that arrives while Replaying; cleared on entering Replaying.
private bool _faultDuringReplay;
/// <summary>
/// True while a reopen or replay is actually running — false between attempts (the backoff delay,
/// during which the state still reads Reopening/Replaying) and while Healthy. #865: an advise waits
/// for the replay's pump only while one is imminent.
/// </summary>
public bool AttemptInProgress => Volatile.Read(ref _attemptInProgress);
/// <summary>True when CurrentState != Healthy. Drivers map this to DriverState.Degraded.</summary>
public bool IsDegraded
{
@@ -136,10 +148,16 @@ public sealed class ReconnectSupervisor : IDisposable
lock (_stateLock)
{
// Re-checked under the lock (StopAsync sets it under the same lock): a report that passed the
// check above while a stop ran must not start a loop nothing would ever cancel.
ObjectDisposedException.ThrowIf(_disposed, this);
_lastError = cause.Message;
if (_state != State.Healthy)
{
// Already recovering — nothing else to do.
// Already recovering — nothing else to do, except: a failure DURING the replay means the
// replay ran onto a transport that is already gone again (#865 — e.g. the restarted event
// stream refused or ended); the replay's success must not then be reported as Healthy.
if (_state == State.Replaying) _faultDuringReplay = true;
_logger.LogDebug("Transport failure reported during {State}: {Message}", _state, cause.Message);
return;
}
@@ -208,20 +226,32 @@ public sealed class ReconnectSupervisor : IDisposable
// === Reopening phase ===
lock (_stateLock) TransitionLocked(State.Reopening, _lastError);
// In progress from the reopen through the replay (no gap between the phases); cleared on any
// exit from the attempt — failure (then the backoff), cancellation or success.
Volatile.Write(ref _attemptInProgress, true);
try
{
await _reopen(ct).ConfigureAwait(false);
}
catch (OperationCanceledException) when (ct.IsCancellationRequested) { return; }
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
Volatile.Write(ref _attemptInProgress, false);
return;
}
catch (Exception ex)
{
Volatile.Write(ref _attemptInProgress, false);
_logger.LogWarning(ex, "Galaxy reopen failed (attempt {Attempt})", attempt);
lock (_stateLock) { _lastError = ex.Message; }
continue; // back to backoff + retry
}
// === Replaying phase ===
lock (_stateLock) TransitionLocked(State.Replaying, _lastError);
lock (_stateLock)
{
_faultDuringReplay = false;
TransitionLocked(State.Replaying, _lastError);
}
try
{
@@ -234,12 +264,32 @@ public sealed class ReconnectSupervisor : IDisposable
lock (_stateLock) { _lastError = ex.Message; }
continue; // back to backoff + retry
}
finally
{
Volatile.Write(ref _attemptInProgress, false);
}
// === Done ===
// === Done — unless the transport failed again while the replay ran ===
bool faultedDuringReplay;
string? replayFault;
lock (_stateLock)
{
_lastError = null;
TransitionLocked(State.Healthy, null);
faultedDuringReplay = _faultDuringReplay;
replayFault = _lastError;
if (!faultedDuringReplay)
{
_lastError = null;
TransitionLocked(State.Healthy, null);
}
}
if (faultedDuringReplay)
{
// Going Healthy here would hide a dead transport (the fault was swallowed above), and the
// next report would restart at attempt 1 with no backoff. Retry this cycle with backoff.
_logger.LogWarning(
"Galaxy replay completed but the transport failed again while it ran (attempt {Attempt}: {Fault}) — retrying",
attempt, replayFault);
continue;
}
_logger.LogInformation("Galaxy reconnect succeeded after {Attempt} attempt(s)", attempt);
return;
@@ -282,12 +332,14 @@ public sealed class ReconnectSupervisor : IDisposable
/// <returns>A task that completes once the recovery loop has exited.</returns>
public async Task StopAsync()
{
if (_disposed) return;
_disposed = true;
CancellationTokenSource? cts;
Task? loop;
lock (_stateLock) { cts = _loopCts; loop = _recoveryLoop; _loopCts = null; _recoveryLoop = null; }
lock (_stateLock)
{
if (_disposed) return;
_disposed = true;
cts = _loopCts; loop = _recoveryLoop; _loopCts = null; _recoveryLoop = null;
}
// The loop's own finally may have released the CTS already — a cancelled-then-disposed CTS
// throws on Cancel, and a disposed one is a no-op on Dispose.
src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy/Runtime/SubscriptionRegistry.cs
+12 -1
View File
@@ -314,6 +314,12 @@ internal sealed class SubscriptionRegistry
/// </summary>
internal Action? AfterMapWriteForTests { get; set; }
/// <summary>
/// Test seam (#865): invoked on <see cref="ResolveOrPark"/>'s lock-free fast path after the handle
/// lookup missed, before the open-window check.
/// </summary>
internal Action? AfterFastPathMissForTests { get; set; }
/// <summary>
/// Test seam (M-1): parks <paramref name="ev"/> unconditionally, producing the parked list for a
/// resolved handle that the atomic registration otherwise makes impossible.
@@ -496,7 +502,12 @@ internal sealed class SubscriptionRegistry
{
subscribers = ResolveSubscribers(ev.ItemHandle);
if (subscribers.Count > 0) return EarlyEventOutcome.Resolved;
if (Volatile.Read(ref _pendingSubscribes) == 0) return EarlyEventOutcome.Stale;
AfterFastPathMissForTests?.Invoke();
// #865 (found by its stress run): a miss is NOT decided here. The lookup above and an
// open-window check are not atomic — a subscribe can register the handle AND close its window
// in between, and declaring the event stale then lost its initial value for good (the first
// event of a bulk advise, while the dispatch thread was slow). Misses are rare; the locked path
// below re-does the lookup, the session check and the window check atomically.
}
lock (_parkGate)