fix(galaxy): #859 final review — probe replay per platform session, teardown session bump

BLOCKER: the reconnect replay chose work from the registry entries' sessions,
but ONE registry entry holds every probe handle and any probe publish on the
new session (a seed in the reopen→replay gap, or the straddle retry) re-stamps
it as current — the replay then skipped the probe watcher and the old
platforms stayed bound to handle 0 (host status frozen); a publish after the
snapshot made it re-subscribe everything (double advise, leaked handle).
ReplayAsync now excludes the probe entry from its snapshot and calls the new
PerPlatformProbeWatcher.ResubscribeStaleAsync on every reconnect, which
re-subscribes exactly the platforms whose ProbeHandle.Session predates the
current session. The replay-gap path keeps ResubscribeAsync (all platforms).

Also: ClearRegistry (teardown) starts a new registry session, so a late
PurgeParked from a teardown-orphaned call cannot reach the next runtime's
parked values; the read path's unused pump local is gone (call kept).

Tests go through the driver's reopen/replay: the probe old-session-number test
is rewritten that way, plus the seed-in-the-gap scenario (only P and Q are
re-subscribed, R keeps its new-session handle, all three resolve). Both checked
red with the probe replay removed / re-subscribing all platforms.

Refs #859, #861.

Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
Joseph Doherty
2026-11-11 11:07:59 -05:00
parent 133bc07da5
commit a8d3790b09
3 changed files with 136 additions and 20 deletions
@@ -690,7 +690,13 @@ public sealed class GalaxyDriver
private void ClearRegistry()
{
lock (_registryGate) _subscriptions.Clear();
lock (_registryGate)
{
_subscriptions.Clear();
// A new session too: a late PurgeParked from a call the teardown orphaned (captured the old
// session) must never reach the next runtime's parked values.
_subscriptions.StartNewSession();
}
}
/// <summary>
@@ -1036,27 +1042,49 @@ public sealed class GalaxyDriver
// Only subscriptions still holding the previous session's handles: one registered after the
// reopen is already live on the recreated session (re-subscribing it would double-advise).
var entries = _subscriptions.SnapshotEntriesNotIn(_subscriptions.CurrentSession);
if (entries.Count == 0) return;
// The probe entry is excluded: ONE registry entry holds every probe handle and any probe publish
// on the new session re-stamps it, so its session says nothing about which platforms are stale.
// The probe watcher tracks the session per handle and is driven separately below.
var session = _subscriptions.CurrentSession;
var entries = _subscriptions.SnapshotEntriesNotIn(session)
.Where(e => e.SubscriptionId != SubscriptionRegistry.ProbeSubscriptionId)
.ToArray();
var probeWatcher = _probeWatcher;
if (entries.Length == 0 && probeWatcher is null) return;
// Honor ReplayOnSessionLost. When operators opt out (false) the per-tag SubscribeBulk fan-out
// is skipped, and the recreated gateway session carries NONE of the old advises — those
// subscriptions stay deaf (their handles are dead and no longer resolve) until re-subscribed.
if (!_options.Reconnect.ReplayOnSessionLost)
{
_logger.LogWarning(
"GalaxyDriver {InstanceId} reconnect replay skipped — ReplayOnSessionLost=false; {SubCount} subscription(s) " +
"receive no further data until they are re-subscribed (the recreated gateway session has none of their advises).",
_driverInstanceId, entries.Count);
if (entries.Length > 0)
{
_logger.LogWarning(
"GalaxyDriver {InstanceId} reconnect replay skipped — ReplayOnSessionLost=false; {SubCount} subscription(s) " +
"receive no further data until they are re-subscribed (the recreated gateway session has none of their advises).",
_driverInstanceId, entries.Length);
}
return;
}
var tagCount = await ResubscribeAllAsync(entries, Volatile.Read(ref _runtimeGeneration), cancellationToken)
.ConfigureAwait(false);
var generation = Volatile.Read(ref _runtimeGeneration);
var tagCount = entries.Length == 0
? 0
: await ResubscribeAllAsync(entries, generation, cancellationToken).ConfigureAwait(false);
// Probes on EVERY reconnect, whatever the registry entry's session: re-subscribes exactly the
// platforms whose handles predate the current session (a platform a seed already advised on the
// new session is left alone — no double advise), and republishes the fenced bindings.
if (probeWatcher is not null
&& Volatile.Read(ref _runtimeGeneration) == generation
&& _subscriptions.CurrentSession == session)
{
tagCount += await probeWatcher.ResubscribeStaleAsync(cancellationToken).ConfigureAwait(false);
}
_logger.LogInformation(
"GalaxyDriver {InstanceId} replay completed — {SubCount} subscriptions, {TagCount} tags re-subscribed",
_driverInstanceId, entries.Count, tagCount);
_driverInstanceId, entries.Length, tagCount);
}
/// <summary>
@@ -1555,7 +1583,9 @@ public sealed class GalaxyDriver
var generation = Volatile.Read(ref _runtimeGeneration); // see SubscribeAsync
var readSubscriber = _subscriber!;
var pump = EnsureEventPumpStarted();
// A live pump must exist (values arrive through it); the read listens on ReadDataChange, not
// on this instance, so it survives a reconnect's pump restart.
EnsureEventPumpStarted();
var subscriptionId = _subscriptions.NextSubscriptionId();
// Pre-allocate one TaskCompletionSource per full-reference so the OnDataChange
@@ -150,6 +150,30 @@ public sealed class PerPlatformProbeWatcher : IDisposable
return platforms.Length;
}
/// <summary>
/// The reconnect replay: re-issue SubscribeBulk only for platforms whose handle predates the
/// registry's current session, then republish. A platform a seed already advised on the new
/// session is left alone (re-subscribing it would double-advise and leak a handle), and every
/// reconnect calls this regardless of the shared registry entry's session — that entry is
/// re-stamped by any probe publish, so it cannot say which platforms are stale (#859).
/// </summary>
/// <param name="cancellationToken">A cancellation token to cancel the operation.</param>
/// <returns>The number of probe references re-subscribed.</returns>
public async Task<int> ResubscribeStaleAsync(CancellationToken cancellationToken)
{
ObjectDisposedException.ThrowIf(_disposed, this);
string[] stale;
lock (_syncLock)
{
var session = CurrentSession;
stale = [.. _itemHandlesByPlatform.Where(kvp => kvp.Value.Session != session).Select(kvp => kvp.Key)];
}
if (stale.Length == 0) return 0;
await SubscribeAndRecordAsync(stale, cancellationToken).ConfigureAwait(false);
return stale.Length;
}
private async Task SubscribeAndRecordAsync(IReadOnlyList<string> platforms, CancellationToken cancellationToken)
{
// #861 / #859 I-A: a reply that straddled a gateway reconnect carries dead handles; re-issue.