fix(galaxy): #859/#861 third review — read rebind release, probe session fence, reopen fence, pump restart
B-1: the subscribe-once read's cleanup unsubscribes what the registry holds at removal (the replay may have rebound it to new-session handles), not its original bindings; nothing when the entry still holds dead old-session handles. Found while testing it: reads were attached to the pump INSTANCE, so a read across a reconnect's pump restart never saw the replayed value — reads now listen on a driver-level ReadDataChange raised for whichever pump is live. I-A: PerPlatformProbeWatcher records the registry session with each probe handle. Publishing goes through the new fenced TryUpsert and binds only current-session handles; removal/stop unsubscribe and purge only those; a seed reply that straddled a reconnect records nothing and is re-issued (max 3). I-B: PurgeParked takes the session captured BEFORE the unsubscribe RPC and is a no-op under the park gate if the session changed. I-C: ReopenAsync starts a new registry session before AND after the gateway session swap; ReplayAsync does so only when no reopen ran (injected subscriber / tests) and replays only subscriptions still holding older-session handles, so a subscribe issued after the swap is live and not double-advised. P-1: ReplayAsync restarts the pump before any early return (no entries used to leave the dead pump in place). P-2: ReplayOnSessionLost=false logs a Warning; misleading comment about gateway-side replay fixed. M-a: TryRebind returns BindOutcome; UnknownId releases the handles just advised for a subscription removed mid-replay. M-b: the replay/re-snapshot pass reads the session once and stops if it changes. M-c: parked entries drop their MxEvent when they leave the parked set. M-d (retry a straddled read) not done. Tests (TestWait) for each, each checked red against a mutation of its fix. Refs #859, #861. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
@@ -250,6 +250,9 @@ public sealed class GalaxyDriver
|
||||
/// <summary>Test seam: true while an <see cref="EventPump"/> is running.</summary>
|
||||
internal bool HasEventPumpForTests { get { lock (_pumpLock) return _eventPump is not null; } }
|
||||
|
||||
/// <summary>Test seam — the current pump instance (identity only), to observe a restart.</summary>
|
||||
internal object? EventPumpInstanceForTests { get { lock (_pumpLock) return _eventPump; } }
|
||||
|
||||
/// <summary>Test seam: the number of driver subscriptions the registry still tracks.</summary>
|
||||
internal int TrackedSubscriptionCountForTests => _subscriptions.TrackedSubscriptionCount;
|
||||
|
||||
@@ -671,13 +674,17 @@ public sealed class GalaxyDriver
|
||||
private async Task UnsubscribeAndPurgeAsync(
|
||||
IGalaxySubscriber subscriber, IReadOnlyList<int> itemHandles, CancellationToken cancellationToken)
|
||||
{
|
||||
// Captured BEFORE the RPC (#859 I-B): a slow or failed unsubscribe can finish after a reconnect,
|
||||
// when these numbers may be the NEW session's — the purge is then a no-op instead of deleting
|
||||
// their parked initial values.
|
||||
var session = _subscriptions.CurrentSession;
|
||||
try
|
||||
{
|
||||
await subscriber.UnsubscribeBulkAsync(itemHandles, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_subscriptions.PurgeParked(itemHandles);
|
||||
_subscriptions.PurgeParked(itemHandles, session);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -948,14 +955,49 @@ public sealed class GalaxyDriver
|
||||
/// </summary>
|
||||
private async Task ReopenAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_ownedMxSession is null) return;
|
||||
var clientOptions = await BuildClientOptionsAsync(_options.Gateway, cancellationToken).ConfigureAwait(false);
|
||||
await _ownedMxSession.RecreateAsync(clientOptions, cancellationToken).ConfigureAwait(false);
|
||||
if (_ownedMxSession is null && RecreateOverrideForTests is null) return;
|
||||
|
||||
// #859 I-C: fence the registry around the swap itself, not later in ReplayAsync — a
|
||||
// SubscribeBulk issued between RecreateAsync and the replay goes to the NEW gateway session.
|
||||
// BEFORE: from here on nothing registered so far counts as live, so an UnsubscribeAsync during
|
||||
// reopen/backoff never sends an old number (which the new session may reuse).
|
||||
_subscriptions.StartNewSession();
|
||||
if (RecreateOverrideForTests is { } recreate)
|
||||
{
|
||||
await recreate(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
var clientOptions = await BuildClientOptionsAsync(_options.Gateway, cancellationToken).ConfigureAwait(false);
|
||||
await _ownedMxSession!.RecreateAsync(clientOptions, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
// AFTER: a subscribe that captured the session between the two bumps may have dialled the old
|
||||
// (dead) session, so it is refused and re-issued; one that captures it from here on dials the
|
||||
// recreated session (GatewayGalaxySubscriber reads GalaxyMxSession.Session at call time), so
|
||||
// it registers normally and the replay leaves it alone instead of re-subscribing it.
|
||||
Volatile.Write(ref _sessionOpenedByReopen, _subscriptions.StartNewSession());
|
||||
|
||||
// The recreated session invalidates every prior gw item handle; drop the writer's handle/advise
|
||||
// caches so the next write re-AddItems + re-AdviseSupervisory against the fresh session.
|
||||
_dataWriter?.InvalidateHandleCaches();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The registry session <see cref="ReopenAsync"/> opened for the recreated gateway session, or -1.
|
||||
/// <see cref="ReplayAsync"/> starts a new session itself only when no reopen did (the injected-
|
||||
/// subscriber path, and tests); a second bump after a reopen would fence subscribes that already
|
||||
/// registered against the recreated session.
|
||||
/// </summary>
|
||||
private int _sessionOpenedByReopen = -1;
|
||||
|
||||
/// <summary>Test seam — replaces the gateway session swap inside <see cref="ReopenAsync"/>.</summary>
|
||||
internal Func<CancellationToken, Task>? RecreateOverrideForTests { get; set; }
|
||||
|
||||
/// <summary>Test-visible entry into <see cref="ReopenAsync"/>.</summary>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
internal Task InvokeReopenForTestAsync(CancellationToken cancellationToken) => ReopenAsync(cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Replay callback. Walks every active subscription, re-issues SubscribeBulk for
|
||||
/// its tag list, and <see cref="SubscriptionRegistry.Rebind">rebinds</see> the
|
||||
@@ -977,32 +1019,38 @@ public sealed class GalaxyDriver
|
||||
// may be reused by the new session (#859 I-1 / #861). Invalidate the reverse map (the
|
||||
// subscriptions stay; the per-iteration Rebind below re-binds each one, and new-session
|
||||
// events park meanwhile instead of resolving to the old owner of a reused number), discard
|
||||
// held events, and fence any subscribe whose reply straddled the reconnect. Done FIRST and
|
||||
// unconditionally — before the pump restart and even with nothing to replay.
|
||||
_subscriptions.StartNewSession();
|
||||
|
||||
var entries = _subscriptions.SnapshotEntries();
|
||||
if (entries.Count == 0) return;
|
||||
|
||||
// Honor ReplayOnSessionLost. When operators opt out (false) we skip the
|
||||
// per-tag SubscribeBulk fan-out — they're delegating to the gateway's
|
||||
// session-level ReplaySubscriptions or accept post-reconnect tag loss. We
|
||||
// still restart the EventPump so a future Subscribe call lands on a live
|
||||
// consumer.
|
||||
if (!_options.Reconnect.ReplayOnSessionLost)
|
||||
// held events, and fence any subscribe whose reply straddled the reconnect. ReopenAsync
|
||||
// already did this around the session swap; only when no reopen ran (injected subscriber,
|
||||
// tests) is it done here — FIRST, before the pump restart and even with nothing to replay.
|
||||
if (Interlocked.Exchange(ref _sessionOpenedByReopen, -1) != _subscriptions.CurrentSession)
|
||||
{
|
||||
RestartEventPumpForReplay();
|
||||
_logger.LogInformation(
|
||||
"GalaxyDriver {InstanceId} reconnect replay skipped — ReplayOnSessionLost=false ({SubCount} subscriptions tracked)",
|
||||
_driverInstanceId, entries.Count);
|
||||
return;
|
||||
_subscriptions.StartNewSession();
|
||||
}
|
||||
|
||||
// The stream-fault that triggered this recovery left the old pump's RunAsync loop
|
||||
// exited and its channel completed; EventPump.Start() is a no-op on a non-null but
|
||||
// completed loop. Recreate the pump so the replayed subscriptions have a consumer.
|
||||
// completed loop. Recreate the pump BEFORE anything can return early (#859 P-1): with no
|
||||
// subscriptions to replay the dead pump used to stay in _eventPump, so every later
|
||||
// Subscribe/Read reused it and was silently deaf while health stayed Healthy.
|
||||
RestartEventPumpForReplay();
|
||||
|
||||
// Only subscriptions still holding the previous session's handles: one registered after the
|
||||
// reopen is already live on the recreated session (re-subscribing it would double-advise).
|
||||
var entries = _subscriptions.SnapshotEntriesNotIn(_subscriptions.CurrentSession);
|
||||
if (entries.Count == 0) return;
|
||||
|
||||
// Honor ReplayOnSessionLost. When operators opt out (false) the per-tag SubscribeBulk fan-out
|
||||
// is skipped, and the recreated gateway session carries NONE of the old advises — those
|
||||
// subscriptions stay deaf (their handles are dead and no longer resolve) until re-subscribed.
|
||||
if (!_options.Reconnect.ReplayOnSessionLost)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"GalaxyDriver {InstanceId} reconnect replay skipped — ReplayOnSessionLost=false; {SubCount} subscription(s) " +
|
||||
"receive no further data until they are re-subscribed (the recreated gateway session has none of their advises).",
|
||||
_driverInstanceId, entries.Count);
|
||||
return;
|
||||
}
|
||||
|
||||
var tagCount = await ResubscribeAllAsync(entries, Volatile.Read(ref _runtimeGeneration), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
@@ -1032,6 +1080,9 @@ public sealed class GalaxyDriver
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var tagCount = 0;
|
||||
// M-b: the session this pass belongs to, read ONCE. Any later change fences the rest of the
|
||||
// pass (the new session's own replay takes over).
|
||||
var session = _subscriptions.CurrentSession;
|
||||
foreach (var (subscriptionId, oldBindings) in entries)
|
||||
{
|
||||
if (Volatile.Read(ref _runtimeGeneration) != generation)
|
||||
@@ -1039,6 +1090,7 @@ public sealed class GalaxyDriver
|
||||
_logger.LogDebug("GalaxyDriver {InstanceId} re-subscribe abandoned — runtime was torn down.", _driverInstanceId);
|
||||
return tagCount;
|
||||
}
|
||||
if (_subscriptions.CurrentSession != session) return tagCount; // a newer session's replay owns the rest
|
||||
|
||||
if (subscriptionId == SubscriptionRegistry.ProbeSubscriptionId && _probeWatcher is { } probeWatcher)
|
||||
{
|
||||
@@ -1061,7 +1113,6 @@ public sealed class GalaxyDriver
|
||||
// window held across all N round-trips would park every unrelated unknown-handle event
|
||||
// for the whole replay and could exhaust the bound before the genuine early values.
|
||||
using var earlyEventWindow = _subscriptions.BeginPendingSubscribe();
|
||||
var session = _subscriptions.CurrentSession;
|
||||
var results = await _subscriber!
|
||||
.SubscribeBulkAsync(refs, _options.MxAccess.PublishingIntervalMs, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
@@ -1079,18 +1130,41 @@ public sealed class GalaxyDriver
|
||||
// Rebind so the EventPump fan-out reverse map points at the post-reconnect
|
||||
// handles; otherwise events on the new handles miss every subscription. Not after a
|
||||
// teardown: the fresh handles belong to a subscriber that is already being released.
|
||||
SubscriptionRegistry.BindOutcome outcome;
|
||||
lock (_registryGate)
|
||||
{
|
||||
if (Volatile.Read(ref _runtimeGeneration) != generation) return tagCount;
|
||||
// #861: another reconnect during this round-trip makes these handles dead too; the
|
||||
// newer session's own replay re-subscribes everything, so stop here.
|
||||
if (!_subscriptions.TryRebind(subscriptionId, newBindings, session))
|
||||
outcome = _subscriptions.TryRebind(subscriptionId, newBindings, session);
|
||||
}
|
||||
|
||||
if (outcome == SubscriptionRegistry.BindOutcome.SessionChanged)
|
||||
{
|
||||
// #861: another reconnect during the pass makes these handles dead too; the newer
|
||||
// session's own replay re-subscribes everything, so stop here.
|
||||
_logger.LogDebug(
|
||||
"GalaxyDriver {InstanceId} re-subscribe abandoned — the gateway session changed again.",
|
||||
_driverInstanceId);
|
||||
return tagCount;
|
||||
}
|
||||
if (outcome == SubscriptionRegistry.BindOutcome.UnknownId)
|
||||
{
|
||||
// M-a: the subscription (or a subscribe-once read) was removed while its SubscribeBulk
|
||||
// was in flight. Nothing will ever unsubscribe what was just advised — release it now.
|
||||
var orphaned = newBindings.Where(b => b.ItemHandle > 0).Select(b => b.ItemHandle).ToArray();
|
||||
if (orphaned.Length > 0)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"GalaxyDriver {InstanceId} re-subscribe abandoned — the gateway session changed again.",
|
||||
_driverInstanceId);
|
||||
return tagCount;
|
||||
try
|
||||
{
|
||||
await UnsubscribeAndPurgeAsync(_subscriber!, orphaned, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"GalaxyDriver {InstanceId} could not release {Count} handle(s) re-subscribed for a subscription removed during the pass.",
|
||||
_driverInstanceId, orphaned.Length);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
tagCount += refs.Length;
|
||||
}
|
||||
@@ -1446,6 +1520,13 @@ public sealed class GalaxyDriver
|
||||
return ReadViaSubscribeOnceAsync(dialledRefs, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Raw (attributeRef-domain) data changes for in-flight subscribe-once reads. Raised by
|
||||
/// <see cref="OnPumpDataChange"/> for whichever pump is current, so a read survives a reconnect's
|
||||
/// pump restart.
|
||||
/// </summary>
|
||||
private event EventHandler<DataChangeEventArgs>? ReadDataChange;
|
||||
|
||||
/// <summary>Upper bound on one subscribe-once read; internal so tests can shrink it.</summary>
|
||||
internal TimeSpan ReadViaSubscribeDeadline { get; set; } = TimeSpan.FromSeconds(30);
|
||||
|
||||
@@ -1500,7 +1581,7 @@ public sealed class GalaxyDriver
|
||||
tcs.TrySetResult(args.Snapshot);
|
||||
}
|
||||
};
|
||||
pump.OnDataChange += handler;
|
||||
ReadDataChange += handler;
|
||||
|
||||
var bufferedIntervalMs = _options.MxAccess.PublishingIntervalMs;
|
||||
IReadOnlyList<SubscribeResult> results;
|
||||
@@ -1525,7 +1606,7 @@ public sealed class GalaxyDriver
|
||||
catch (OperationCanceledException) when (!callerToken.IsCancellationRequested)
|
||||
{
|
||||
// Internal deadline fired while the subscribe RPC was still in flight.
|
||||
pump.OnDataChange -= handler;
|
||||
ReadDataChange -= handler;
|
||||
var subscriber = readSubscriber;
|
||||
var logger = _logger;
|
||||
_ = subscribeTask.ContinueWith(t =>
|
||||
@@ -1550,7 +1631,7 @@ public sealed class GalaxyDriver
|
||||
}
|
||||
catch
|
||||
{
|
||||
pump.OnDataChange -= handler;
|
||||
ReadDataChange -= handler;
|
||||
throw;
|
||||
}
|
||||
|
||||
@@ -1584,7 +1665,7 @@ public sealed class GalaxyDriver
|
||||
var registered = TryRegisterForGeneration(subscriptionId, bindings, generation, session);
|
||||
if (registered != RegisterOutcome.Registered)
|
||||
{
|
||||
pump.OnDataChange -= handler;
|
||||
ReadDataChange -= handler;
|
||||
earlyEventWindow.Dispose(); // before any further round-trip
|
||||
if (registered == RegisterOutcome.TornDown)
|
||||
{
|
||||
@@ -1637,16 +1718,20 @@ public sealed class GalaxyDriver
|
||||
}
|
||||
finally
|
||||
{
|
||||
pump.OnDataChange -= handler;
|
||||
ReadDataChange -= handler;
|
||||
// Drop the bindings + unsubscribe the live handles. UnsubscribeBulkAsync's
|
||||
// failure isn't fatal — the registry is already cleared, so any straggling
|
||||
// event from the gw would be a no-op fan-out.
|
||||
var removed = _subscriptions.Remove(subscriptionId, out var liveInCurrentSession);
|
||||
// A reconnect during the wait left these handles in the dead session; their numbers may now
|
||||
// be new-session items, so they must not be unsubscribed (#861).
|
||||
var staleSession = removed is not null && !liveInCurrentSession;
|
||||
var liveHandles = bindings.Where(b => b.ItemHandle > 0).Select(b => b.ItemHandle).ToArray();
|
||||
if (liveHandles.Length > 0 && !staleSession)
|
||||
// Unsubscribe what the registry held at removal, NOT the local `bindings` (#859 B-1): a
|
||||
// reconnect replay during the wait may have rebound this read to new-session handles, and
|
||||
// the original numbers may now be other tags' live items. If the entry still holds dead
|
||||
// old-session handles (not yet rebound), release nothing (#861). Only when the entry was
|
||||
// already gone (teardown cleared the registry) fall back to the original bindings.
|
||||
IReadOnlyList<TagBinding> source = removed ?? bindings;
|
||||
if (removed is not null && !liveInCurrentSession) source = [];
|
||||
var liveHandles = source.Where(b => b.ItemHandle > 0).Select(b => b.ItemHandle).ToArray();
|
||||
if (liveHandles.Length > 0)
|
||||
{
|
||||
try
|
||||
{
|
||||
@@ -2277,6 +2362,14 @@ public sealed class GalaxyDriver
|
||||
// supervisor Healthy transition, so a 90 s Degraded tripped a re-init under flowing data).
|
||||
StampLiveness();
|
||||
|
||||
// Subscribe-once reads listen here (attributeRef domain), not on a pump instance: a reconnect
|
||||
// replaces the pump, and a read attached to the dead one never saw the replayed value (#859 B-1).
|
||||
try { ReadDataChange?.Invoke(this, args); }
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "GalaxyDriver {InstanceId} read listener threw — continuing.", _driverInstanceId);
|
||||
}
|
||||
|
||||
// Probe traffic rides the reserved registry entry and stays in the attributeRef domain —
|
||||
// ScanState attributes are discovered platform attributes, never authored RawPaths. It is
|
||||
// the watcher's alone: the host never subscribed it and must not see it on OnDataChange.
|
||||
|
||||
@@ -46,7 +46,10 @@ public sealed class PerPlatformProbeWatcher : IDisposable
|
||||
|
||||
// Tracked platform → gw item handle. Item handle 0 means the gw rejected the subscribe;
|
||||
// we keep the entry so SyncPlatformsAsync doesn't try to subscribe it again on every call.
|
||||
private readonly ConcurrentDictionary<string, int> _itemHandlesByPlatform =
|
||||
// Each handle carries the registry session that issued it (#859 I-A): item handles are per gateway
|
||||
// session and the next session reuses the numbers, so an old-session handle must never be published,
|
||||
// unsubscribed or purged against the current session.
|
||||
private readonly ConcurrentDictionary<string, ProbeHandle> _itemHandlesByPlatform =
|
||||
new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly Lock _syncLock = new();
|
||||
// Written under _syncLock by StopAsync; read under it wherever a write-back must not outlive a stop.
|
||||
@@ -99,7 +102,14 @@ public sealed class PerPlatformProbeWatcher : IDisposable
|
||||
internal IReadOnlyList<TagBinding> CurrentBindings => BuildBindings();
|
||||
|
||||
private IReadOnlyList<TagBinding> BuildBindings() =>
|
||||
[.. _itemHandlesByPlatform.Select(kvp => new TagBinding(kvp.Key + ProbeSuffix, kvp.Value))];
|
||||
[.. _itemHandlesByPlatform.Select(kvp => new TagBinding(kvp.Key + ProbeSuffix, kvp.Value.ItemHandle))];
|
||||
|
||||
/// <summary>The bindings to publish for <paramref name="session"/>: another session's handles bind as 0 (no events).</summary>
|
||||
private IReadOnlyList<TagBinding> BuildBindingsFor(int session) =>
|
||||
[.. _itemHandlesByPlatform.Select(kvp => new TagBinding(
|
||||
kvp.Key + ProbeSuffix, kvp.Value.Session == session ? kvp.Value.ItemHandle : 0))];
|
||||
|
||||
private int CurrentSession => _registry?.CurrentSession ?? 0;
|
||||
|
||||
/// <summary>
|
||||
/// Mirror the tracked handle set into the driver registry (no-op without one, or once stopped —
|
||||
@@ -107,15 +117,20 @@ public sealed class PerPlatformProbeWatcher : IDisposable
|
||||
/// </summary>
|
||||
private void PublishBindings()
|
||||
{
|
||||
lock (_syncLock) PublishBindingsLocked();
|
||||
lock (_syncLock) PublishBindingsLocked(CurrentSession);
|
||||
}
|
||||
|
||||
private void PublishBindingsLocked()
|
||||
/// <returns>False when the registry's session is no longer <paramref name="session"/> (nothing published).</returns>
|
||||
private bool PublishBindingsLocked(int session)
|
||||
{
|
||||
if (_registry is null || _disposed) return;
|
||||
var bindings = BuildBindings();
|
||||
if (bindings.Count == 0) _registry.Remove(SubscriptionRegistry.ProbeSubscriptionId);
|
||||
else _registry.Upsert(SubscriptionRegistry.ProbeSubscriptionId, bindings);
|
||||
if (_registry is null || _disposed) return true;
|
||||
if (_itemHandlesByPlatform.IsEmpty)
|
||||
{
|
||||
_registry.Remove(SubscriptionRegistry.ProbeSubscriptionId);
|
||||
return true;
|
||||
}
|
||||
// Fenced (#859 I-A): only handles issued by `session` are bound, and only while it is current.
|
||||
return _registry.TryUpsert(SubscriptionRegistry.ProbeSubscriptionId, BuildBindingsFor(session), session);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -136,12 +151,31 @@ public sealed class PerPlatformProbeWatcher : IDisposable
|
||||
}
|
||||
|
||||
private async Task SubscribeAndRecordAsync(IReadOnlyList<string> platforms, CancellationToken cancellationToken)
|
||||
{
|
||||
// #861 / #859 I-A: a reply that straddled a gateway reconnect carries dead handles; re-issue.
|
||||
for (var attempt = 1; ; attempt++)
|
||||
{
|
||||
if (await TrySubscribeAndRecordAsync(platforms, cancellationToken).ConfigureAwait(false)) return;
|
||||
if (attempt >= MaxSessionAttempts)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"PerPlatformProbeWatcher probe subscribe straddled {Attempts} gateway reconnects — giving up until the next replay/seed", attempt);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private const int MaxSessionAttempts = 3;
|
||||
|
||||
/// <returns>False when the reply straddled a session change (nothing recorded; retry).</returns>
|
||||
private async Task<bool> TrySubscribeAndRecordAsync(IReadOnlyList<string> platforms, CancellationToken cancellationToken)
|
||||
{
|
||||
var probeAddresses = platforms.Select(p => p + ProbeSuffix).ToArray();
|
||||
|
||||
// #859: the initial ScanState value can overtake the SubscribeBulk reply; park it until
|
||||
// PublishBindingsLocked registers the handles below (window closes at method exit).
|
||||
using var earlyEventWindow = _registry?.BeginPendingSubscribe();
|
||||
var session = CurrentSession; // the session this SubscribeBulk is issued on
|
||||
// PR 6.3 — use the configured bufferedUpdateIntervalMs (defaults to 0 = gw cadence
|
||||
// when the driver hasn't overridden MxAccess.PublishingIntervalMs). Probe ScanState
|
||||
// changes are rare so a coarser interval is usually fine; deployments that need
|
||||
@@ -167,8 +201,15 @@ public sealed class PerPlatformProbeWatcher : IDisposable
|
||||
stopped = _disposed;
|
||||
if (!stopped)
|
||||
{
|
||||
foreach (var (platform, itemHandle, _) in acquired) _itemHandlesByPlatform[platform] = itemHandle;
|
||||
PublishBindingsLocked();
|
||||
if (_registry is not null && _registry.CurrentSession != session)
|
||||
{
|
||||
// Straddled a reconnect: these handles died with the old session and their numbers may
|
||||
// be reused — neither record, publish nor unsubscribe them.
|
||||
return false;
|
||||
}
|
||||
foreach (var (platform, itemHandle, _) in acquired)
|
||||
_itemHandlesByPlatform[platform] = new ProbeHandle(itemHandle, session);
|
||||
if (!PublishBindingsLocked(session)) return false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -184,7 +225,7 @@ public sealed class PerPlatformProbeWatcher : IDisposable
|
||||
"PerPlatformProbeWatcher could not release {Count} probe handle(s) acquired after stop", orphans.Length);
|
||||
}
|
||||
}
|
||||
return;
|
||||
return true;
|
||||
}
|
||||
|
||||
foreach (var (platform, itemHandle, error) in acquired)
|
||||
@@ -194,6 +235,7 @@ public sealed class PerPlatformProbeWatcher : IDisposable
|
||||
_logger.LogWarning("PerPlatformProbeWatcher subscribe failed for {Platform}: {Error}", platform, error);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -228,12 +270,29 @@ public sealed class PerPlatformProbeWatcher : IDisposable
|
||||
// race on the membership view.
|
||||
List<string> toAdd;
|
||||
List<(string Platform, int ItemHandle)> toRemove;
|
||||
int removeSession;
|
||||
bool droppedWithoutRelease;
|
||||
lock (_syncLock)
|
||||
{
|
||||
toAdd = [.. desired.Where(p => !_itemHandlesByPlatform.ContainsKey(p))];
|
||||
// Captured before the RPC (#859 I-B); only handles issued by the CURRENT session are released —
|
||||
// an older session's numbers may now belong to unrelated live items (#859 I-A).
|
||||
removeSession = CurrentSession;
|
||||
toRemove = [.. _itemHandlesByPlatform
|
||||
.Where(kvp => !desired.Contains(kvp.Key) && kvp.Value > 0)
|
||||
.Select(kvp => (kvp.Key, kvp.Value))];
|
||||
.Where(kvp => !desired.Contains(kvp.Key)
|
||||
&& kvp.Value.ItemHandle > 0 && kvp.Value.Session == removeSession)
|
||||
.Select(kvp => (kvp.Key, kvp.Value.ItemHandle))];
|
||||
var deadRemovals = _itemHandlesByPlatform
|
||||
.Where(kvp => !desired.Contains(kvp.Key) && !(kvp.Value.ItemHandle > 0 && kvp.Value.Session == removeSession))
|
||||
.Select(kvp => kvp.Key).ToArray();
|
||||
// Dropped platforms with no releasable handle (rejected, or an older session's dead number):
|
||||
// forget them without any unsubscribe.
|
||||
foreach (var platform in deadRemovals)
|
||||
{
|
||||
_itemHandlesByPlatform.TryRemove(platform, out _);
|
||||
_aggregator.Remove(platform);
|
||||
}
|
||||
droppedWithoutRelease = deadRemovals.Length > 0;
|
||||
|
||||
// Drop removed entries from the membership map up-front so a concurrent
|
||||
// OnProbeValueChanged for them is silently ignored. The unsubscribe RPC
|
||||
@@ -245,6 +304,8 @@ public sealed class PerPlatformProbeWatcher : IDisposable
|
||||
}
|
||||
}
|
||||
|
||||
if (droppedWithoutRelease && toRemove.Count == 0) PublishBindings(); // keep the fan-out map in step
|
||||
|
||||
if (toRemove.Count > 0)
|
||||
{
|
||||
// Drop the dead handles from the fan-out map before the (slow, fallible) unsubscribe.
|
||||
@@ -264,7 +325,7 @@ public sealed class PerPlatformProbeWatcher : IDisposable
|
||||
{
|
||||
// #859 I-3 interim: no trailing event for a removed probe handle may be promoted onto
|
||||
// a reused handle number.
|
||||
_registry?.PurgeParked([.. toRemove.Select(t => t.ItemHandle)]);
|
||||
_registry?.PurgeParked([.. toRemove.Select(t => t.ItemHandle)], removeSession);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -334,7 +395,9 @@ public sealed class PerPlatformProbeWatcher : IDisposable
|
||||
{
|
||||
if (_disposed) return;
|
||||
_disposed = true;
|
||||
liveHandles = [.. _itemHandlesByPlatform.Values.Where(h => h > 0)];
|
||||
var session = CurrentSession; // never release an older session's (reusable) numbers
|
||||
liveHandles = [.. _itemHandlesByPlatform.Values
|
||||
.Where(h => h.ItemHandle > 0 && h.Session == session).Select(h => h.ItemHandle)];
|
||||
_itemHandlesByPlatform.Clear();
|
||||
_registry?.Remove(SubscriptionRegistry.ProbeSubscriptionId);
|
||||
}
|
||||
@@ -347,4 +410,9 @@ public sealed class PerPlatformProbeWatcher : IDisposable
|
||||
|
||||
/// <summary>Disposes the probe watcher and unsubscribes all tracked platforms (blocking on the round-trip).</summary>
|
||||
public void Dispose() => StopAsync(CancellationToken.None).GetAwaiter().GetResult();
|
||||
|
||||
/// <summary>A probe item handle and the registry session that issued it.</summary>
|
||||
/// <param name="ItemHandle">The gateway item handle (0 = rejected).</param>
|
||||
/// <param name="Session">The registry session the handle belongs to.</param>
|
||||
private readonly record struct ProbeHandle(int ItemHandle, int Session);
|
||||
}
|
||||
|
||||
@@ -83,7 +83,7 @@ internal sealed class SubscriptionRegistry
|
||||
/// <param name="expectedSession">The <see cref="CurrentSession"/> captured before the SubscribeBulk.</param>
|
||||
/// <returns>False (nothing registered) when the session changed in the meantime.</returns>
|
||||
public bool TryRegister(long subscriptionId, IReadOnlyList<TagBinding> bindings, int expectedSession) =>
|
||||
Bind(subscriptionId, bindings, requireExisting: false, expectedSession) == BindResult.Bound;
|
||||
Bind(subscriptionId, bindings, requireExisting: false, expectedSession) == BindOutcome.Bound;
|
||||
|
||||
/// <summary>
|
||||
/// Replace an existing subscription's bindings with the item handles a post-reconnect
|
||||
@@ -100,13 +100,36 @@ internal sealed class SubscriptionRegistry
|
||||
/// <param name="subscriptionId">The subscription identifier.</param>
|
||||
/// <param name="newBindings">The new tag bindings.</param>
|
||||
/// <param name="expectedSession">The session the SubscribeBulk was issued on.</param>
|
||||
/// <returns>False when the session changed (nothing rebound); true otherwise, including the unknown-id no-op.</returns>
|
||||
public bool TryRebind(long subscriptionId, IReadOnlyList<TagBinding> newBindings, int expectedSession) =>
|
||||
Bind(subscriptionId, newBindings, requireExisting: true, expectedSession) != BindResult.SessionChanged;
|
||||
/// <returns>
|
||||
/// <see cref="BindOutcome.Bound"/>; <see cref="BindOutcome.UnknownId"/> when the subscription was
|
||||
/// removed meanwhile (the caller must release the handles it just acquired — nothing will
|
||||
/// ever unsubscribe them, M-a); <see cref="BindOutcome.SessionChanged"/> when the session moved on.
|
||||
/// </returns>
|
||||
public BindOutcome TryRebind(long subscriptionId, IReadOnlyList<TagBinding> newBindings, int expectedSession) =>
|
||||
Bind(subscriptionId, newBindings, requireExisting: true, expectedSession);
|
||||
|
||||
private enum BindResult { Bound, UnknownId, SessionChanged }
|
||||
/// <summary>
|
||||
/// <see cref="Upsert"/> fenced by <paramref name="expectedSession"/> (the probe watcher, #859 I-A).
|
||||
/// </summary>
|
||||
/// <param name="subscriptionId">The subscription identifier.</param>
|
||||
/// <param name="bindings">The tag bindings, all issued by <paramref name="expectedSession"/>.</param>
|
||||
/// <param name="expectedSession">The session the bindings' handles belong to.</param>
|
||||
/// <returns>False when the session changed (nothing published).</returns>
|
||||
public bool TryUpsert(long subscriptionId, IReadOnlyList<TagBinding> bindings, int expectedSession) =>
|
||||
Bind(subscriptionId, bindings, requireExisting: false, expectedSession) == BindOutcome.Bound;
|
||||
|
||||
private BindResult Bind(
|
||||
/// <summary>Outcome of a fenced bind.</summary>
|
||||
public enum BindOutcome
|
||||
{
|
||||
/// <summary>Bound.</summary>
|
||||
Bound,
|
||||
/// <summary>The subscription id is no longer tracked (rebind only).</summary>
|
||||
UnknownId,
|
||||
/// <summary>The gateway session changed since the handles were issued.</summary>
|
||||
SessionChanged,
|
||||
}
|
||||
|
||||
private BindOutcome Bind(
|
||||
long subscriptionId, IReadOnlyList<TagBinding> bindings, bool requireExisting, int? expectedSession)
|
||||
{
|
||||
bool promoted;
|
||||
@@ -117,7 +140,7 @@ internal sealed class SubscriptionRegistry
|
||||
// session check is in here too, so it is atomic with StartNewSession's invalidation.
|
||||
lock (_parkGate)
|
||||
{
|
||||
if (expectedSession is { } session && session != _session) return BindResult.SessionChanged;
|
||||
if (expectedSession is { } session && session != _session) return BindOutcome.SessionChanged;
|
||||
|
||||
if (_bySubscriptionId.TryGetValue(subscriptionId, out var oldEntry))
|
||||
{
|
||||
@@ -128,7 +151,7 @@ internal sealed class SubscriptionRegistry
|
||||
}
|
||||
else if (requireExisting)
|
||||
{
|
||||
return BindResult.UnknownId;
|
||||
return BindOutcome.UnknownId;
|
||||
}
|
||||
|
||||
_bySubscriptionId[subscriptionId] = new SubscriptionEntry(subscriptionId, bindings, _session);
|
||||
@@ -144,7 +167,7 @@ internal sealed class SubscriptionRegistry
|
||||
promoted = PromoteParkedEventsLocked(bindings);
|
||||
}
|
||||
if (promoted) ParkedEventsReady?.Invoke();
|
||||
return BindResult.Bound;
|
||||
return BindOutcome.Bound;
|
||||
}
|
||||
|
||||
private void RemoveFromReverseMap(SubscriptionEntry entry)
|
||||
@@ -397,7 +420,7 @@ internal sealed class SubscriptionRegistry
|
||||
// head of its handle's queue.
|
||||
var queue = _parkedByItemHandle[head.ItemHandle];
|
||||
queue.Dequeue();
|
||||
head.Gone = true;
|
||||
head.Release();
|
||||
if (queue.Count == 0) _parkedByItemHandle.Remove(head.ItemHandle);
|
||||
Volatile.Write(ref _heldCount, _heldCount - 1);
|
||||
}
|
||||
@@ -405,7 +428,7 @@ internal sealed class SubscriptionRegistry
|
||||
|
||||
private void DiscardAllHeldLocked()
|
||||
{
|
||||
foreach (var parked in _parkOrder) parked.Gone = true;
|
||||
foreach (var parked in _parkOrder) parked.Release();
|
||||
_parkOrder.Clear();
|
||||
_parkedByItemHandle.Clear();
|
||||
_ready.Clear();
|
||||
@@ -423,15 +446,21 @@ internal sealed class SubscriptionRegistry
|
||||
/// worker sequence on the SubscribeBulk reply (mxaccessgw, cross-repo).
|
||||
/// </summary>
|
||||
/// <param name="itemHandles">The handles whose unsubscribe completed.</param>
|
||||
public void PurgeParked(IReadOnlyList<int> itemHandles)
|
||||
/// <param name="session">
|
||||
/// The session the handles (and the unsubscribe) belong to, captured BEFORE the RPC. A slow or
|
||||
/// failed unsubscribe can complete after a reconnect; purging then would delete the NEW
|
||||
/// session's parked initial values for reused numbers, so it is a no-op (#859 I-B).
|
||||
/// </param>
|
||||
public void PurgeParked(IReadOnlyList<int> itemHandles, int session)
|
||||
{
|
||||
if (Volatile.Read(ref _heldCount) == 0 || itemHandles.Count == 0) return;
|
||||
lock (_parkGate)
|
||||
{
|
||||
if (session != _session) return;
|
||||
foreach (var handle in itemHandles)
|
||||
{
|
||||
if (!_parkedByItemHandle.Remove(handle, out var queue)) continue;
|
||||
foreach (var parked in queue) parked.Gone = true;
|
||||
foreach (var parked in queue) parked.Release();
|
||||
Volatile.Write(ref _heldCount, _heldCount - queue.Count);
|
||||
}
|
||||
}
|
||||
@@ -489,8 +518,7 @@ internal sealed class SubscriptionRegistry
|
||||
{
|
||||
foreach (var parked in stranded)
|
||||
{
|
||||
parked.Gone = true;
|
||||
_ready.Enqueue(parked.Event);
|
||||
_ready.Enqueue(parked.Release());
|
||||
}
|
||||
}
|
||||
replayFirst = TakeReadyLocked();
|
||||
@@ -548,8 +576,7 @@ internal sealed class SubscriptionRegistry
|
||||
if (!_parkedByItemHandle.Remove(binding.ItemHandle, out var queue)) continue;
|
||||
foreach (var parked in queue)
|
||||
{
|
||||
parked.Gone = true; // stays counted in _heldCount until replayed
|
||||
_ready.Enqueue(parked.Event);
|
||||
_ready.Enqueue(parked.Release()); // stays counted in _heldCount until replayed
|
||||
}
|
||||
promoted = true;
|
||||
}
|
||||
@@ -562,10 +589,19 @@ internal sealed class SubscriptionRegistry
|
||||
public long Stamp { get; } = stamp;
|
||||
/// <summary>Gets the event's item handle.</summary>
|
||||
public int ItemHandle { get; } = itemHandle;
|
||||
/// <summary>Gets the event.</summary>
|
||||
public MxEvent Event { get; } = ev;
|
||||
/// <summary>Gets or sets whether the event has left the parked set (promoted, purged or trimmed).</summary>
|
||||
public bool Gone { get; set; }
|
||||
/// <summary>Gets the event; null once it has left the parked set (M-c — not pinned by the trim index).</summary>
|
||||
public MxEvent? Event { get; private set; } = ev;
|
||||
/// <summary>Gets whether the event has left the parked set (promoted, purged or trimmed).</summary>
|
||||
public bool Gone => Event is null;
|
||||
|
||||
/// <summary>Takes the event out of this entry, so the trim index no longer pins it.</summary>
|
||||
/// <returns>The event.</returns>
|
||||
public MxEvent Release()
|
||||
{
|
||||
var ev = Event!;
|
||||
Event = null;
|
||||
return ev;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class PendingSubscribeScope(SubscriptionRegistry owner, long sequence) : IDisposable
|
||||
@@ -593,6 +629,16 @@ internal sealed class SubscriptionRegistry
|
||||
public IReadOnlyList<(long SubscriptionId, IReadOnlyList<TagBinding> Bindings)> SnapshotEntries() =>
|
||||
[.. _bySubscriptionId.Values.Select(entry => (entry.SubscriptionId, entry.Bindings))];
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="SnapshotEntries"/> limited to subscriptions whose handles were NOT issued by
|
||||
/// <paramref name="session"/> — the reconnect replay's work list. A subscription registered after
|
||||
/// the reconnect already holds live handles; re-subscribing it would double-advise (#859 I-C).
|
||||
/// </summary>
|
||||
/// <param name="session">The current session.</param>
|
||||
/// <returns>The stale subscriptions with their bindings.</returns>
|
||||
public IReadOnlyList<(long SubscriptionId, IReadOnlyList<TagBinding> Bindings)> SnapshotEntriesNotIn(int session) =>
|
||||
[.. _bySubscriptionId.Values.Where(e => e.Session != session).Select(e => (e.SubscriptionId, e.Bindings))];
|
||||
|
||||
/// <summary>
|
||||
/// Per-subscription bookkeeping. <see cref="FullRefByItemHandle"/> is an index
|
||||
/// over <see cref="Bindings"/> keyed by item handle so <c>ResolveSubscribers</c>
|
||||
|
||||
Reference in New Issue
Block a user