fix(galaxy): #859/#861 third review — read rebind release, probe session fence, reopen fence, pump restart

B-1: the subscribe-once read's cleanup unsubscribes what the registry holds at
removal (the replay may have rebound it to new-session handles), not its
original bindings; nothing when the entry still holds dead old-session
handles. Found while testing it: reads were attached to the pump INSTANCE, so
a read across a reconnect's pump restart never saw the replayed value — reads
now listen on a driver-level ReadDataChange raised for whichever pump is live.

I-A: PerPlatformProbeWatcher records the registry session with each probe
handle. Publishing goes through the new fenced TryUpsert and binds only
current-session handles; removal/stop unsubscribe and purge only those; a
seed reply that straddled a reconnect records nothing and is re-issued (max 3).

I-B: PurgeParked takes the session captured BEFORE the unsubscribe RPC and is
a no-op under the park gate if the session changed.

I-C: ReopenAsync starts a new registry session before AND after the gateway
session swap; ReplayAsync does so only when no reopen ran (injected subscriber
/ tests) and replays only subscriptions still holding older-session handles,
so a subscribe issued after the swap is live and not double-advised.

P-1: ReplayAsync restarts the pump before any early return (no entries used to
leave the dead pump in place). P-2: ReplayOnSessionLost=false logs a Warning;
misleading comment about gateway-side replay fixed.

M-a: TryRebind returns BindOutcome; UnknownId releases the handles just
advised for a subscription removed mid-replay. M-b: the replay/re-snapshot
pass reads the session once and stops if it changes. M-c: parked entries drop
their MxEvent when they leave the parked set. M-d (retry a straddled read)
not done.

Tests (TestWait) for each, each checked red against a mutation of its fix.

Refs #859, #861.

Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
Joseph Doherty
2026-11-11 11:02:55 -05:00
parent d08e900203
commit 133bc07da5
5 changed files with 481 additions and 83 deletions
@@ -210,7 +210,7 @@ public sealed class EarlyEventWindowTests
registry.ResolveOrPark(DataChange(7, 1), session, out _, out _); // trailing event of a removed item
registry.ResolveOrPark(DataChange(8, 2), session, out _, out _); // positive control
registry.PurgeParked([7]);
registry.PurgeParked([7], session);
registry.ParkedEventCount.ShouldBe(1);
registry.Register(1, [new TagBinding("Reused", 7), new TagBinding("Other", 8)]);
@@ -297,7 +297,7 @@ public sealed class EarlyEventWindowTests
using var window = registry.BeginPendingSubscribe();
// New session: Old.B is re-issued as handle 7 — the number Old.A still holds until its own Rebind.
registry.ResolveOrPark(DataChange(7, 5), session, out _, out _).ShouldBe(EarlyEventOutcome.Parked);
registry.TryRebind(2, [new TagBinding("Old.B", 7)], session).ShouldBeTrue();
registry.TryRebind(2, [new TagBinding("Old.B", 7)], session).ShouldBe(SubscriptionRegistry.BindOutcome.Bound);
registry.TakeReadyEvents(session).ShouldHaveSingleItem().Value.DoubleValue.ShouldBe(5.0);
registry.ResolveSubscribers(7).ShouldBe([(2L, "Old.B")]);
@@ -314,7 +314,7 @@ public sealed class EarlyEventWindowTests
registry.TryRegister(2, [new TagBinding("Late", 9)], before).ShouldBeFalse();
registry.Contains(2).ShouldBeFalse();
registry.TryRebind(1, [new TagBinding("Tank.Level", 9)], before).ShouldBeFalse();
registry.TryRebind(1, [new TagBinding("Tank.Level", 9)], before).ShouldBe(SubscriptionRegistry.BindOutcome.SessionChanged);
registry.ResolveSubscribers(9).ShouldBeEmpty();
registry.TryRegister(3, [new TagBinding("Fresh", 9)], registry.CurrentSession).ShouldBeTrue("positive control");
@@ -683,9 +683,10 @@ public sealed class EarlyEventWindowTests
subscriber.Script.Enqueue(async (refs, _) =>
{
var tag = refs[0];
var before = registry.ParkedEventCount;
// The previous iteration's promoted value stays counted until the pump replays it.
await TestWait.UntilAsync(() => registry.ParkedEventCount == 0, "earlier promoted values must be replayed first");
subscriber.Emit(DataChange(newHandle[tag], newValue[tag]));
await TestWait.UntilAsync(() => registry.ParkedEventCount > before,
await TestWait.UntilAsync(() => registry.ParkedEventCount > 0,
$"{tag}'s new-session initial value must PARK, not resolve to the old owner of handle {newHandle[tag]}");
return [Ok(tag, newHandle[tag])];
});
@@ -778,4 +779,194 @@ public sealed class EarlyEventWindowTests
driver.SubscriptionsForTests.PendingSubscribeCount.ShouldBe(0);
never.SetResult([]); // let the abandoned RPC finish cleanly
}
// ===== Third review (B-1, I-A, I-B, I-C, P-1, M-a) =====
/// <summary>
/// B-1: a read rebound by a reconnect replay while it waited unsubscribes its NEW handle, not the
/// original number (which may by now be another tag's live item).
/// </summary>
[Fact]
public async Task Read_rebound_by_a_replay_unsubscribes_the_new_handle_not_the_original()
{
var subscriber = new ScriptedSubscriber();
using var driver = new GalaxyDriver(
"g", Opts(), hierarchySource: null, dataReader: null, dataWriter: null, subscriber: subscriber);
subscriber.Script.Enqueue((refs, _) => Task.FromResult<IReadOnlyList<SubscribeResult>>([Ok(refs[0], 1)]));
subscriber.Script.Enqueue((refs, _) =>
{
subscriber.Emit(DataChange(9, 99.0)); // the replayed initial value (may overtake the reply)
return Task.FromResult<IReadOnlyList<SubscribeResult>>([Ok(refs[0], 9)]);
});
var read = driver.ReadAsync(["Tank.R"], TestContext.Current.CancellationToken);
await TestWait.UntilAsync(() => driver.TrackedSubscriptionCountForTests == 1, "the read must be registered and waiting");
await driver.InvokeReplayForTestAsync(TestContext.Current.CancellationToken);
var snapshot = (await TestWait.ForAsync(read, "the replayed value must complete the read")).ShouldHaveSingleItem();
((double)snapshot.Value!).ShouldBe(99.0);
subscriber.Unsubscribed.ToArray().ShouldBe([9], "the read must release its NEW handle, never the original number 1");
}
/// <summary>P-1: a reconnect with nothing to replay still replaces the dead pump; a later subscribe gets data.</summary>
[Fact]
public async Task Reconnect_with_no_subscriptions_restarts_the_pump()
{
var subscriber = new ScriptedSubscriber();
using var driver = new GalaxyDriver(
"g", Opts(), hierarchySource: null, dataReader: null, dataWriter: null, subscriber: subscriber);
var delivered = new ConcurrentQueue<DataChangeEventArgs>();
driver.OnDataChange += (_, e) => delivered.Enqueue(e);
var first = await driver.SubscribeAsync(["Tank.Old"], TimeSpan.Zero, TestContext.Current.CancellationToken);
await driver.UnsubscribeAsync(first, TestContext.Current.CancellationToken);
var deadPump = driver.EventPumpInstanceForTests.ShouldNotBeNull();
await driver.InvokeReplayForTestAsync(TestContext.Current.CancellationToken);
driver.EventPumpInstanceForTests.ShouldNotBeSameAs(deadPump, "the faulted pump must be replaced even with no entries");
subscriber.Script.Enqueue((refs, _) => Task.FromResult<IReadOnlyList<SubscribeResult>>([Ok(refs[0], 42)]));
await driver.SubscribeAsync(["Tank.New"], TimeSpan.Zero, TestContext.Current.CancellationToken);
subscriber.Emit(DataChange(42, 4.2));
await TestWait.UntilAsync(() => !delivered.IsEmpty, "a subscribe after the reconnect must receive data");
delivered.Single().FullReference.ShouldBe("Tank.New");
}
/// <summary>
/// I-C: the reopen fences the registry around the session swap. An unsubscribe during the
/// reopen/backoff never sends an old number; a subscribe issued after the swap registers normally
/// and the replay does not re-subscribe it (no double advise).
/// </summary>
[Fact]
public async Task Subscribe_after_the_session_swap_but_before_replay_is_live_and_not_replayed()
{
var subscriber = new ScriptedSubscriber();
using var driver = new GalaxyDriver(
"g", Opts(), hierarchySource: null, dataReader: null, dataWriter: null, subscriber: subscriber)
{
RecreateOverrideForTests = _ => Task.CompletedTask,
};
var delivered = new ConcurrentQueue<(string Ref, double Value)>();
driver.OnDataChange += (_, e) => delivered.Enqueue((e.FullReference, (double)e.Snapshot.Value!));
subscriber.Script.Enqueue((refs, _) => Task.FromResult<IReadOnlyList<SubscribeResult>>([Ok(refs[0], 1)]));
subscriber.Script.Enqueue((refs, _) => Task.FromResult<IReadOnlyList<SubscribeResult>>([Ok(refs[0], 3)]));
await driver.SubscribeAsync(["Tank.A"], TimeSpan.Zero, TestContext.Current.CancellationToken);
var c = await driver.SubscribeAsync(["Tank.C"], TimeSpan.Zero, TestContext.Current.CancellationToken);
await driver.InvokeReopenForTestAsync(TestContext.Current.CancellationToken); // gateway session swapped
await driver.UnsubscribeAsync(c, TestContext.Current.CancellationToken);
subscriber.Unsubscribed.ShouldNotContain(3, "an old-session number must never reach the new session");
subscriber.Script.Enqueue((refs, _) => Task.FromResult<IReadOnlyList<SubscribeResult>>([Ok(refs[0], 50)]));
await driver.SubscribeAsync(["Tank.B"], TimeSpan.Zero, TestContext.Current.CancellationToken);
var callsBeforeReplay = Volatile.Read(ref subscriber.SubscribeCalls);
subscriber.Script.Enqueue((refs, _) => Task.FromResult<IReadOnlyList<SubscribeResult>>([Ok(refs[0], 51)]));
await driver.InvokeReplayForTestAsync(TestContext.Current.CancellationToken);
(Volatile.Read(ref subscriber.SubscribeCalls) - callsBeforeReplay).ShouldBe(1, "only Tank.A (old session) is re-subscribed");
var registry = driver.SubscriptionsForTests;
registry.ResolveSubscribers(50).Single().FullReference.ShouldBe("Tank.B");
registry.ResolveSubscribers(51).Single().FullReference.ShouldBe("Tank.A");
subscriber.Emit(DataChange(50, 5.0));
await TestWait.UntilAsync(() => !delivered.IsEmpty, "the post-swap subscription must stay live across the replay");
delivered.Single().ShouldBe(("Tank.B", 5.0));
}
/// <summary>
/// M-a: a subscription removed while the replay's SubscribeBulk for it was in flight gets the
/// freshly advised handles released (nothing else would ever unsubscribe them).
/// </summary>
[Fact]
public async Task Replay_releases_handles_for_a_subscription_removed_mid_replay()
{
var subscriber = new ScriptedSubscriber();
using var driver = new GalaxyDriver(
"g", Opts(), hierarchySource: null, dataReader: null, dataWriter: null, subscriber: subscriber);
subscriber.Script.Enqueue((refs, _) => Task.FromResult<IReadOnlyList<SubscribeResult>>([Ok(refs[0], 1)]));
var handle = await driver.SubscribeAsync(["Tank.Gone"], TimeSpan.Zero, TestContext.Current.CancellationToken);
subscriber.Script.Enqueue(async (refs, _) =>
{
await driver.UnsubscribeAsync(handle, CancellationToken.None); // removed mid-replay
return [Ok(refs[0], 77)];
});
await driver.InvokeReplayForTestAsync(TestContext.Current.CancellationToken);
subscriber.Unsubscribed.ToArray().ShouldBe([77], "the re-advised handle is released; the dead number 1 is not sent");
driver.SubscriptionsForTests.ResolveSubscribers(77).ShouldBeEmpty();
}
/// <summary>I-B: a purge that completes after a reconnect does not delete the new session's parked values.</summary>
[Fact]
public void A_purge_for_an_older_session_is_a_no_op()
{
var registry = new SubscriptionRegistry();
var oldSession = registry.CurrentSession;
var newSession = registry.StartNewSession();
using var window = registry.BeginPendingSubscribe();
registry.ResolveOrPark(DataChange(7, 1), newSession, out _, out _).ShouldBe(EarlyEventOutcome.Parked);
registry.PurgeParked([7], oldSession); // a slow old-session unsubscribe finishing late
registry.ParkedEventCount.ShouldBe(1, "the new session's parked initial value must survive");
registry.PurgeParked([7], newSession); // positive control
registry.ParkedEventCount.ShouldBe(0);
}
/// <summary>I-A: a probe seed whose reply straddles a reconnect records nothing dead and is re-issued.</summary>
[Fact]
public async Task Probe_seed_straddling_a_reconnect_is_reissued_and_never_publishes_the_dead_handle()
{
var registry = new SubscriptionRegistry();
var subscriber = new ScriptedSubscriber();
var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
subscriber.Script.Enqueue(async (refs, _) =>
{
await gate.Task;
return [Ok(refs[0], 5)]; // old-session handle
});
subscriber.Script.Enqueue((refs, _) => Task.FromResult<IReadOnlyList<SubscribeResult>>([Ok(refs[0], 6)]));
using var watcher = new PerPlatformProbeWatcher(subscriber, new HostStatusAggregator(), registry);
var sync = watcher.SyncPlatformsAsync(["P"], TestContext.Current.CancellationToken);
await TestWait.UntilAsync(() => Volatile.Read(ref subscriber.SubscribeCalls) == 1, "the seed's SubscribeBulk must be in flight");
registry.StartNewSession();
gate.SetResult();
await TestWait.ForAsync(sync, "the seed must complete");
Volatile.Read(ref subscriber.SubscribeCalls).ShouldBe(2);
watcher.CurrentBindings.Single().ItemHandle.ShouldBe(6);
registry.ResolveSubscribers(5).ShouldBeEmpty();
registry.ResolveSubscribers(6).Single().FullReference.ShouldBe("P.ScanState");
subscriber.Unsubscribed.ShouldBeEmpty();
}
/// <summary>
/// I-A: after a reconnect (before the replay rebinds probes), a seed neither re-publishes the old
/// session's probe numbers nor unsubscribes a removed platform's old number.
/// </summary>
[Fact]
public async Task Probe_watcher_never_publishes_or_unsubscribes_old_session_numbers()
{
var registry = new SubscriptionRegistry();
var subscriber = new ScriptedSubscriber();
subscriber.Script.Enqueue((refs, _) => Task.FromResult<IReadOnlyList<SubscribeResult>>(
[.. refs.Select((r, i) => Ok(r, i + 1))])); // P → 1, Q → 2
using var watcher = new PerPlatformProbeWatcher(subscriber, new HostStatusAggregator(), registry);
await watcher.SyncPlatformsAsync(["P", "Q"], TestContext.Current.CancellationToken);
registry.StartNewSession();
// The new session reuses number 1 for a user tag.
registry.Register(10, [new TagBinding("User.Tag", 1)]);
await watcher.SyncPlatformsAsync(["P"], TestContext.Current.CancellationToken); // drops Q
subscriber.Unsubscribed.ShouldBeEmpty("Q's handle 2 is an old-session number");
registry.ResolveSubscribers(1).ShouldBe([(10L, "User.Tag")], "P's old-session number must not be re-published");
(await watcher.ResubscribeAsync(TestContext.Current.CancellationToken)).ShouldBe(1); // the replay
var fresh = watcher.CurrentBindings.Single().ItemHandle;
registry.ResolveSubscribers(fresh).Single().FullReference.ShouldBe("P.ScanState");
}
}