fix(galaxy): #859/#861 third review — read rebind release, probe session fence, reopen fence, pump restart
B-1: the subscribe-once read's cleanup unsubscribes what the registry holds at removal (the replay may have rebound it to new-session handles), not its original bindings; nothing when the entry still holds dead old-session handles. Found while testing it: reads were attached to the pump INSTANCE, so a read across a reconnect's pump restart never saw the replayed value — reads now listen on a driver-level ReadDataChange raised for whichever pump is live. I-A: PerPlatformProbeWatcher records the registry session with each probe handle. Publishing goes through the new fenced TryUpsert and binds only current-session handles; removal/stop unsubscribe and purge only those; a seed reply that straddled a reconnect records nothing and is re-issued (max 3). I-B: PurgeParked takes the session captured BEFORE the unsubscribe RPC and is a no-op under the park gate if the session changed. I-C: ReopenAsync starts a new registry session before AND after the gateway session swap; ReplayAsync does so only when no reopen ran (injected subscriber / tests) and replays only subscriptions still holding older-session handles, so a subscribe issued after the swap is live and not double-advised. P-1: ReplayAsync restarts the pump before any early return (no entries used to leave the dead pump in place). P-2: ReplayOnSessionLost=false logs a Warning; misleading comment about gateway-side replay fixed. M-a: TryRebind returns BindOutcome; UnknownId releases the handles just advised for a subscription removed mid-replay. M-b: the replay/re-snapshot pass reads the session once and stops if it changes. M-c: parked entries drop their MxEvent when they leave the parked set. M-d (retry a straddled read) not done. Tests (TestWait) for each, each checked red against a mutation of its fix. Refs #859, #861. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
+196
-5
@@ -210,7 +210,7 @@ public sealed class EarlyEventWindowTests
|
||||
registry.ResolveOrPark(DataChange(7, 1), session, out _, out _); // trailing event of a removed item
|
||||
registry.ResolveOrPark(DataChange(8, 2), session, out _, out _); // positive control
|
||||
|
||||
registry.PurgeParked([7]);
|
||||
registry.PurgeParked([7], session);
|
||||
|
||||
registry.ParkedEventCount.ShouldBe(1);
|
||||
registry.Register(1, [new TagBinding("Reused", 7), new TagBinding("Other", 8)]);
|
||||
@@ -297,7 +297,7 @@ public sealed class EarlyEventWindowTests
|
||||
using var window = registry.BeginPendingSubscribe();
|
||||
// New session: Old.B is re-issued as handle 7 — the number Old.A still holds until its own Rebind.
|
||||
registry.ResolveOrPark(DataChange(7, 5), session, out _, out _).ShouldBe(EarlyEventOutcome.Parked);
|
||||
registry.TryRebind(2, [new TagBinding("Old.B", 7)], session).ShouldBeTrue();
|
||||
registry.TryRebind(2, [new TagBinding("Old.B", 7)], session).ShouldBe(SubscriptionRegistry.BindOutcome.Bound);
|
||||
|
||||
registry.TakeReadyEvents(session).ShouldHaveSingleItem().Value.DoubleValue.ShouldBe(5.0);
|
||||
registry.ResolveSubscribers(7).ShouldBe([(2L, "Old.B")]);
|
||||
@@ -314,7 +314,7 @@ public sealed class EarlyEventWindowTests
|
||||
|
||||
registry.TryRegister(2, [new TagBinding("Late", 9)], before).ShouldBeFalse();
|
||||
registry.Contains(2).ShouldBeFalse();
|
||||
registry.TryRebind(1, [new TagBinding("Tank.Level", 9)], before).ShouldBeFalse();
|
||||
registry.TryRebind(1, [new TagBinding("Tank.Level", 9)], before).ShouldBe(SubscriptionRegistry.BindOutcome.SessionChanged);
|
||||
registry.ResolveSubscribers(9).ShouldBeEmpty();
|
||||
registry.TryRegister(3, [new TagBinding("Fresh", 9)], registry.CurrentSession).ShouldBeTrue("positive control");
|
||||
|
||||
@@ -683,9 +683,10 @@ public sealed class EarlyEventWindowTests
|
||||
subscriber.Script.Enqueue(async (refs, _) =>
|
||||
{
|
||||
var tag = refs[0];
|
||||
var before = registry.ParkedEventCount;
|
||||
// The previous iteration's promoted value stays counted until the pump replays it.
|
||||
await TestWait.UntilAsync(() => registry.ParkedEventCount == 0, "earlier promoted values must be replayed first");
|
||||
subscriber.Emit(DataChange(newHandle[tag], newValue[tag]));
|
||||
await TestWait.UntilAsync(() => registry.ParkedEventCount > before,
|
||||
await TestWait.UntilAsync(() => registry.ParkedEventCount > 0,
|
||||
$"{tag}'s new-session initial value must PARK, not resolve to the old owner of handle {newHandle[tag]}");
|
||||
return [Ok(tag, newHandle[tag])];
|
||||
});
|
||||
@@ -778,4 +779,194 @@ public sealed class EarlyEventWindowTests
|
||||
driver.SubscriptionsForTests.PendingSubscribeCount.ShouldBe(0);
|
||||
never.SetResult([]); // let the abandoned RPC finish cleanly
|
||||
}
|
||||
|
||||
// ===== Third review (B-1, I-A, I-B, I-C, P-1, M-a) =====
|
||||
|
||||
/// <summary>
|
||||
/// B-1: a read rebound by a reconnect replay while it waited unsubscribes its NEW handle, not the
|
||||
/// original number (which may by now be another tag's live item).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Read_rebound_by_a_replay_unsubscribes_the_new_handle_not_the_original()
|
||||
{
|
||||
var subscriber = new ScriptedSubscriber();
|
||||
using var driver = new GalaxyDriver(
|
||||
"g", Opts(), hierarchySource: null, dataReader: null, dataWriter: null, subscriber: subscriber);
|
||||
subscriber.Script.Enqueue((refs, _) => Task.FromResult<IReadOnlyList<SubscribeResult>>([Ok(refs[0], 1)]));
|
||||
subscriber.Script.Enqueue((refs, _) =>
|
||||
{
|
||||
subscriber.Emit(DataChange(9, 99.0)); // the replayed initial value (may overtake the reply)
|
||||
return Task.FromResult<IReadOnlyList<SubscribeResult>>([Ok(refs[0], 9)]);
|
||||
});
|
||||
|
||||
var read = driver.ReadAsync(["Tank.R"], TestContext.Current.CancellationToken);
|
||||
await TestWait.UntilAsync(() => driver.TrackedSubscriptionCountForTests == 1, "the read must be registered and waiting");
|
||||
await driver.InvokeReplayForTestAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
var snapshot = (await TestWait.ForAsync(read, "the replayed value must complete the read")).ShouldHaveSingleItem();
|
||||
((double)snapshot.Value!).ShouldBe(99.0);
|
||||
subscriber.Unsubscribed.ToArray().ShouldBe([9], "the read must release its NEW handle, never the original number 1");
|
||||
}
|
||||
|
||||
/// <summary>P-1: a reconnect with nothing to replay still replaces the dead pump; a later subscribe gets data.</summary>
|
||||
[Fact]
|
||||
public async Task Reconnect_with_no_subscriptions_restarts_the_pump()
|
||||
{
|
||||
var subscriber = new ScriptedSubscriber();
|
||||
using var driver = new GalaxyDriver(
|
||||
"g", Opts(), hierarchySource: null, dataReader: null, dataWriter: null, subscriber: subscriber);
|
||||
var delivered = new ConcurrentQueue<DataChangeEventArgs>();
|
||||
driver.OnDataChange += (_, e) => delivered.Enqueue(e);
|
||||
var first = await driver.SubscribeAsync(["Tank.Old"], TimeSpan.Zero, TestContext.Current.CancellationToken);
|
||||
await driver.UnsubscribeAsync(first, TestContext.Current.CancellationToken);
|
||||
var deadPump = driver.EventPumpInstanceForTests.ShouldNotBeNull();
|
||||
|
||||
await driver.InvokeReplayForTestAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
driver.EventPumpInstanceForTests.ShouldNotBeSameAs(deadPump, "the faulted pump must be replaced even with no entries");
|
||||
subscriber.Script.Enqueue((refs, _) => Task.FromResult<IReadOnlyList<SubscribeResult>>([Ok(refs[0], 42)]));
|
||||
await driver.SubscribeAsync(["Tank.New"], TimeSpan.Zero, TestContext.Current.CancellationToken);
|
||||
subscriber.Emit(DataChange(42, 4.2));
|
||||
await TestWait.UntilAsync(() => !delivered.IsEmpty, "a subscribe after the reconnect must receive data");
|
||||
delivered.Single().FullReference.ShouldBe("Tank.New");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// I-C: the reopen fences the registry around the session swap. An unsubscribe during the
|
||||
/// reopen/backoff never sends an old number; a subscribe issued after the swap registers normally
|
||||
/// and the replay does not re-subscribe it (no double advise).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Subscribe_after_the_session_swap_but_before_replay_is_live_and_not_replayed()
|
||||
{
|
||||
var subscriber = new ScriptedSubscriber();
|
||||
using var driver = new GalaxyDriver(
|
||||
"g", Opts(), hierarchySource: null, dataReader: null, dataWriter: null, subscriber: subscriber)
|
||||
{
|
||||
RecreateOverrideForTests = _ => Task.CompletedTask,
|
||||
};
|
||||
var delivered = new ConcurrentQueue<(string Ref, double Value)>();
|
||||
driver.OnDataChange += (_, e) => delivered.Enqueue((e.FullReference, (double)e.Snapshot.Value!));
|
||||
subscriber.Script.Enqueue((refs, _) => Task.FromResult<IReadOnlyList<SubscribeResult>>([Ok(refs[0], 1)]));
|
||||
subscriber.Script.Enqueue((refs, _) => Task.FromResult<IReadOnlyList<SubscribeResult>>([Ok(refs[0], 3)]));
|
||||
await driver.SubscribeAsync(["Tank.A"], TimeSpan.Zero, TestContext.Current.CancellationToken);
|
||||
var c = await driver.SubscribeAsync(["Tank.C"], TimeSpan.Zero, TestContext.Current.CancellationToken);
|
||||
|
||||
await driver.InvokeReopenForTestAsync(TestContext.Current.CancellationToken); // gateway session swapped
|
||||
|
||||
await driver.UnsubscribeAsync(c, TestContext.Current.CancellationToken);
|
||||
subscriber.Unsubscribed.ShouldNotContain(3, "an old-session number must never reach the new session");
|
||||
|
||||
subscriber.Script.Enqueue((refs, _) => Task.FromResult<IReadOnlyList<SubscribeResult>>([Ok(refs[0], 50)]));
|
||||
await driver.SubscribeAsync(["Tank.B"], TimeSpan.Zero, TestContext.Current.CancellationToken);
|
||||
var callsBeforeReplay = Volatile.Read(ref subscriber.SubscribeCalls);
|
||||
|
||||
subscriber.Script.Enqueue((refs, _) => Task.FromResult<IReadOnlyList<SubscribeResult>>([Ok(refs[0], 51)]));
|
||||
await driver.InvokeReplayForTestAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
(Volatile.Read(ref subscriber.SubscribeCalls) - callsBeforeReplay).ShouldBe(1, "only Tank.A (old session) is re-subscribed");
|
||||
var registry = driver.SubscriptionsForTests;
|
||||
registry.ResolveSubscribers(50).Single().FullReference.ShouldBe("Tank.B");
|
||||
registry.ResolveSubscribers(51).Single().FullReference.ShouldBe("Tank.A");
|
||||
subscriber.Emit(DataChange(50, 5.0));
|
||||
await TestWait.UntilAsync(() => !delivered.IsEmpty, "the post-swap subscription must stay live across the replay");
|
||||
delivered.Single().ShouldBe(("Tank.B", 5.0));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// M-a: a subscription removed while the replay's SubscribeBulk for it was in flight gets the
|
||||
/// freshly advised handles released (nothing else would ever unsubscribe them).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Replay_releases_handles_for_a_subscription_removed_mid_replay()
|
||||
{
|
||||
var subscriber = new ScriptedSubscriber();
|
||||
using var driver = new GalaxyDriver(
|
||||
"g", Opts(), hierarchySource: null, dataReader: null, dataWriter: null, subscriber: subscriber);
|
||||
subscriber.Script.Enqueue((refs, _) => Task.FromResult<IReadOnlyList<SubscribeResult>>([Ok(refs[0], 1)]));
|
||||
var handle = await driver.SubscribeAsync(["Tank.Gone"], TimeSpan.Zero, TestContext.Current.CancellationToken);
|
||||
|
||||
subscriber.Script.Enqueue(async (refs, _) =>
|
||||
{
|
||||
await driver.UnsubscribeAsync(handle, CancellationToken.None); // removed mid-replay
|
||||
return [Ok(refs[0], 77)];
|
||||
});
|
||||
await driver.InvokeReplayForTestAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
subscriber.Unsubscribed.ToArray().ShouldBe([77], "the re-advised handle is released; the dead number 1 is not sent");
|
||||
driver.SubscriptionsForTests.ResolveSubscribers(77).ShouldBeEmpty();
|
||||
}
|
||||
|
||||
/// <summary>I-B: a purge that completes after a reconnect does not delete the new session's parked values.</summary>
|
||||
[Fact]
|
||||
public void A_purge_for_an_older_session_is_a_no_op()
|
||||
{
|
||||
var registry = new SubscriptionRegistry();
|
||||
var oldSession = registry.CurrentSession;
|
||||
var newSession = registry.StartNewSession();
|
||||
using var window = registry.BeginPendingSubscribe();
|
||||
registry.ResolveOrPark(DataChange(7, 1), newSession, out _, out _).ShouldBe(EarlyEventOutcome.Parked);
|
||||
|
||||
registry.PurgeParked([7], oldSession); // a slow old-session unsubscribe finishing late
|
||||
registry.ParkedEventCount.ShouldBe(1, "the new session's parked initial value must survive");
|
||||
|
||||
registry.PurgeParked([7], newSession); // positive control
|
||||
registry.ParkedEventCount.ShouldBe(0);
|
||||
}
|
||||
|
||||
/// <summary>I-A: a probe seed whose reply straddles a reconnect records nothing dead and is re-issued.</summary>
|
||||
[Fact]
|
||||
public async Task Probe_seed_straddling_a_reconnect_is_reissued_and_never_publishes_the_dead_handle()
|
||||
{
|
||||
var registry = new SubscriptionRegistry();
|
||||
var subscriber = new ScriptedSubscriber();
|
||||
var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
subscriber.Script.Enqueue(async (refs, _) =>
|
||||
{
|
||||
await gate.Task;
|
||||
return [Ok(refs[0], 5)]; // old-session handle
|
||||
});
|
||||
subscriber.Script.Enqueue((refs, _) => Task.FromResult<IReadOnlyList<SubscribeResult>>([Ok(refs[0], 6)]));
|
||||
using var watcher = new PerPlatformProbeWatcher(subscriber, new HostStatusAggregator(), registry);
|
||||
|
||||
var sync = watcher.SyncPlatformsAsync(["P"], TestContext.Current.CancellationToken);
|
||||
await TestWait.UntilAsync(() => Volatile.Read(ref subscriber.SubscribeCalls) == 1, "the seed's SubscribeBulk must be in flight");
|
||||
registry.StartNewSession();
|
||||
gate.SetResult();
|
||||
await TestWait.ForAsync(sync, "the seed must complete");
|
||||
|
||||
Volatile.Read(ref subscriber.SubscribeCalls).ShouldBe(2);
|
||||
watcher.CurrentBindings.Single().ItemHandle.ShouldBe(6);
|
||||
registry.ResolveSubscribers(5).ShouldBeEmpty();
|
||||
registry.ResolveSubscribers(6).Single().FullReference.ShouldBe("P.ScanState");
|
||||
subscriber.Unsubscribed.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// I-A: after a reconnect (before the replay rebinds probes), a seed neither re-publishes the old
|
||||
/// session's probe numbers nor unsubscribes a removed platform's old number.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Probe_watcher_never_publishes_or_unsubscribes_old_session_numbers()
|
||||
{
|
||||
var registry = new SubscriptionRegistry();
|
||||
var subscriber = new ScriptedSubscriber();
|
||||
subscriber.Script.Enqueue((refs, _) => Task.FromResult<IReadOnlyList<SubscribeResult>>(
|
||||
[.. refs.Select((r, i) => Ok(r, i + 1))])); // P → 1, Q → 2
|
||||
using var watcher = new PerPlatformProbeWatcher(subscriber, new HostStatusAggregator(), registry);
|
||||
await watcher.SyncPlatformsAsync(["P", "Q"], TestContext.Current.CancellationToken);
|
||||
|
||||
registry.StartNewSession();
|
||||
// The new session reuses number 1 for a user tag.
|
||||
registry.Register(10, [new TagBinding("User.Tag", 1)]);
|
||||
|
||||
await watcher.SyncPlatformsAsync(["P"], TestContext.Current.CancellationToken); // drops Q
|
||||
|
||||
subscriber.Unsubscribed.ShouldBeEmpty("Q's handle 2 is an old-session number");
|
||||
registry.ResolveSubscribers(1).ShouldBe([(10L, "User.Tag")], "P's old-session number must not be re-published");
|
||||
|
||||
(await watcher.ResubscribeAsync(TestContext.Current.CancellationToken)).ShouldBe(1); // the replay
|
||||
var fresh = watcher.CurrentBindings.Single().ItemHandle;
|
||||
registry.ResolveSubscribers(fresh).Single().FullReference.ShouldBe("P.ScanState");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user