fix(galaxy): #859 final review — probe replay per platform session, teardown session bump
BLOCKER: the reconnect replay chose work from the registry entries' sessions, but ONE registry entry holds every probe handle and any probe publish on the new session (a seed in the reopen→replay gap, or the straddle retry) re-stamps it as current — the replay then skipped the probe watcher and the old platforms stayed bound to handle 0 (host status frozen); a publish after the snapshot made it re-subscribe everything (double advise, leaked handle). ReplayAsync now excludes the probe entry from its snapshot and calls the new PerPlatformProbeWatcher.ResubscribeStaleAsync on every reconnect, which re-subscribes exactly the platforms whose ProbeHandle.Session predates the current session. The replay-gap path keeps ResubscribeAsync (all platforms). Also: ClearRegistry (teardown) starts a new registry session, so a late PurgeParked from a teardown-orphaned call cannot reach the next runtime's parked values; the read path's unused pump local is gone (call kept). Tests go through the driver's reopen/replay: the probe old-session-number test is rewritten that way, plus the seed-in-the-gap scenario (only P and Q are re-subscribed, R keeps its new-session handle, all three resolve). Both checked red with the probe replay removed / re-subscribing all platforms. Refs #859, #861. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
@@ -690,7 +690,13 @@ public sealed class GalaxyDriver
|
||||
|
||||
private void ClearRegistry()
|
||||
{
|
||||
lock (_registryGate) _subscriptions.Clear();
|
||||
lock (_registryGate)
|
||||
{
|
||||
_subscriptions.Clear();
|
||||
// A new session too: a late PurgeParked from a call the teardown orphaned (captured the old
|
||||
// session) must never reach the next runtime's parked values.
|
||||
_subscriptions.StartNewSession();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -1036,27 +1042,49 @@ public sealed class GalaxyDriver
|
||||
|
||||
// Only subscriptions still holding the previous session's handles: one registered after the
|
||||
// reopen is already live on the recreated session (re-subscribing it would double-advise).
|
||||
var entries = _subscriptions.SnapshotEntriesNotIn(_subscriptions.CurrentSession);
|
||||
if (entries.Count == 0) return;
|
||||
// The probe entry is excluded: ONE registry entry holds every probe handle and any probe publish
|
||||
// on the new session re-stamps it, so its session says nothing about which platforms are stale.
|
||||
// The probe watcher tracks the session per handle and is driven separately below.
|
||||
var session = _subscriptions.CurrentSession;
|
||||
var entries = _subscriptions.SnapshotEntriesNotIn(session)
|
||||
.Where(e => e.SubscriptionId != SubscriptionRegistry.ProbeSubscriptionId)
|
||||
.ToArray();
|
||||
var probeWatcher = _probeWatcher;
|
||||
if (entries.Length == 0 && probeWatcher is null) return;
|
||||
|
||||
// Honor ReplayOnSessionLost. When operators opt out (false) the per-tag SubscribeBulk fan-out
|
||||
// is skipped, and the recreated gateway session carries NONE of the old advises — those
|
||||
// subscriptions stay deaf (their handles are dead and no longer resolve) until re-subscribed.
|
||||
if (!_options.Reconnect.ReplayOnSessionLost)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"GalaxyDriver {InstanceId} reconnect replay skipped — ReplayOnSessionLost=false; {SubCount} subscription(s) " +
|
||||
"receive no further data until they are re-subscribed (the recreated gateway session has none of their advises).",
|
||||
_driverInstanceId, entries.Count);
|
||||
if (entries.Length > 0)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"GalaxyDriver {InstanceId} reconnect replay skipped — ReplayOnSessionLost=false; {SubCount} subscription(s) " +
|
||||
"receive no further data until they are re-subscribed (the recreated gateway session has none of their advises).",
|
||||
_driverInstanceId, entries.Length);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
var tagCount = await ResubscribeAllAsync(entries, Volatile.Read(ref _runtimeGeneration), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
var generation = Volatile.Read(ref _runtimeGeneration);
|
||||
var tagCount = entries.Length == 0
|
||||
? 0
|
||||
: await ResubscribeAllAsync(entries, generation, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Probes on EVERY reconnect, whatever the registry entry's session: re-subscribes exactly the
|
||||
// platforms whose handles predate the current session (a platform a seed already advised on the
|
||||
// new session is left alone — no double advise), and republishes the fenced bindings.
|
||||
if (probeWatcher is not null
|
||||
&& Volatile.Read(ref _runtimeGeneration) == generation
|
||||
&& _subscriptions.CurrentSession == session)
|
||||
{
|
||||
tagCount += await probeWatcher.ResubscribeStaleAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"GalaxyDriver {InstanceId} replay completed — {SubCount} subscriptions, {TagCount} tags re-subscribed",
|
||||
_driverInstanceId, entries.Count, tagCount);
|
||||
_driverInstanceId, entries.Length, tagCount);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -1555,7 +1583,9 @@ public sealed class GalaxyDriver
|
||||
|
||||
var generation = Volatile.Read(ref _runtimeGeneration); // see SubscribeAsync
|
||||
var readSubscriber = _subscriber!;
|
||||
var pump = EnsureEventPumpStarted();
|
||||
// A live pump must exist (values arrive through it); the read listens on ReadDataChange, not
|
||||
// on this instance, so it survives a reconnect's pump restart.
|
||||
EnsureEventPumpStarted();
|
||||
var subscriptionId = _subscriptions.NextSubscriptionId();
|
||||
|
||||
// Pre-allocate one TaskCompletionSource per full-reference so the OnDataChange
|
||||
|
||||
@@ -150,6 +150,30 @@ public sealed class PerPlatformProbeWatcher : IDisposable
|
||||
return platforms.Length;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The reconnect replay: re-issue SubscribeBulk only for platforms whose handle predates the
|
||||
/// registry's current session, then republish. A platform a seed already advised on the new
|
||||
/// session is left alone (re-subscribing it would double-advise and leak a handle), and every
|
||||
/// reconnect calls this regardless of the shared registry entry's session — that entry is
|
||||
/// re-stamped by any probe publish, so it cannot say which platforms are stale (#859).
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>The number of probe references re-subscribed.</returns>
|
||||
public async Task<int> ResubscribeStaleAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
ObjectDisposedException.ThrowIf(_disposed, this);
|
||||
string[] stale;
|
||||
lock (_syncLock)
|
||||
{
|
||||
var session = CurrentSession;
|
||||
stale = [.. _itemHandlesByPlatform.Where(kvp => kvp.Value.Session != session).Select(kvp => kvp.Key)];
|
||||
}
|
||||
if (stale.Length == 0) return 0;
|
||||
|
||||
await SubscribeAndRecordAsync(stale, cancellationToken).ConfigureAwait(false);
|
||||
return stale.Length;
|
||||
}
|
||||
|
||||
private async Task SubscribeAndRecordAsync(IReadOnlyList<string> platforms, CancellationToken cancellationToken)
|
||||
{
|
||||
// #861 / #859 I-A: a reply that straddled a gateway reconnect carries dead handles; re-issue.
|
||||
|
||||
Reference in New Issue
Block a user