fix(galaxy): #865 review round 3 - budget-safe stream wait, dead pump, fast-path drop

- StreamRequestWait 1 s (under Tier-A Read 2 s / Subscribe 5 s budgets) and
  a stale pump is only waited on while the supervisor is reopening or
  replaying (or a restart is underway); in a reconnect backoff the advise
  proceeds at once. The no-stream log is a rate-limited Warning while
  Healthy, Debug otherwise.
- A current-session pump whose loop has ended is not a stream: if the
  supervisor already reports Healthy (its fault was swallowed while it was
  recovering) the failure is re-reported to start a new cycle.
- Subscribe/Read no longer start the pump directly (only
  AwaitCurrentEventStreamAsync does, honouring the replay restart flag);
  the teardown refusal is an explicit ThrowIfRuntimeStopping.
- SubscriptionRegistry.ResolveOrPark fast path: the handle lookup and the
  open-window check were not atomic, so a subscribe that registered the
  handle and closed its window in between got its initial value dropped as
  stale (found by a stress run: the first event of a bulk advise lost). It
  re-looks-up before declaring stale. Deterministic test via a seam.
- Tests: first Read on a fresh driver, replay of a data subscription.

Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
Joseph Doherty
2026-11-11 13:00:19 -05:00
parent 9060ff5561
commit 1c1705fa77
4 changed files with 189 additions and 20 deletions
@@ -898,6 +898,31 @@ public sealed class EarlyEventWindowTests
driver.SubscriptionsForTests.ResolveSubscribers(77).ShouldBeEmpty();
}
/// <summary>
/// #865 stress finding: the dispatch fast path looked the handle up, then checked for an open window —
/// not atomically. A subscribe that registered the handle AND closed its window in between made the
/// event "stale" and its initial value was dropped for good. It must resolve.
/// </summary>
[Fact]
public void A_registration_landing_between_the_fast_path_lookup_and_the_window_check_still_resolves()
{
var registry = new SubscriptionRegistry();
var session = registry.CurrentSession;
var window = registry.BeginPendingSubscribe();
registry.AfterFastPathMissForTests = () =>
{
registry.AfterFastPathMissForTests = null;
registry.TryRegister(registry.NextSubscriptionId(), [new TagBinding("Tank.A", 7)], session).ShouldBeTrue();
window.Dispose(); // the subscribe's window closes right after its registration
};
registry.ResolveOrPark(DataChange(7, 1), session, out var subscribers, out _).ShouldBe(EarlyEventOutcome.Resolved);
subscribers.Single().FullReference.ShouldBe("Tank.A");
// Positive control for the Stale branch: an unknown handle with no window open is still dropped.
registry.ResolveOrPark(DataChange(8, 1), session, out _, out _).ShouldBe(EarlyEventOutcome.Stale);
}
/// <summary>I-B: a purge that completes after a reconnect does not delete the new session's parked values.</summary>
[Fact]
public void A_purge_for_an_older_session_is_a_no_op()
@@ -265,11 +265,56 @@ public sealed class ProbeInitialValueTests
(await TestWait.ForAsync(delivered.Task, "the subscribe's initial value must arrive", () => Describe(driver, sub)))
.FullReference.ShouldBe("Tank01.Level");
sub.DroppedInitialValues.ShouldBe(0);
}
/// <summary>A first ReadAsync on a fresh driver (no pump yet) requests the stream before its advise.</summary>
[Fact]
public async Task First_read_advises_only_after_the_stream_is_requested()
{
var sub = new GatewayLikeSubscriber();
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub)
{
ReadViaSubscribeDeadline = TimeSpan.FromSeconds(10),
};
await driver.InitializeAsync("{}", CancellationToken.None);
driver.HasEventPumpForTests.ShouldBeFalse("precondition: the read is the first advise");
var read = await driver.ReadAsync(["Tank02.Level"], CancellationToken.None);
var calls = sub.CallsSnapshot();
Array.IndexOf(calls, "stream").ShouldBe(0, $"the stream must be requested before the read's advise; calls=[{string.Join(" | ", calls)}]");
read[0].StatusCode.ShouldBe(0u, "the read's initial value must arrive (not BadTimeout)");
sub.DroppedInitialValues.ShouldBe(0);
}
/// <summary>
/// The replay's re-advise of DATA subscriptions (which has no per-advise hook) waits for the restarted
/// pump's stream, so each re-advised tag's new initial value arrives. The value flips so only a
/// post-replay event can satisfy the wait.
/// </summary>
[Fact]
public async Task Reconnect_replay_delivers_re_advised_data_subscriptions_initial_values()
{
var sub = new GatewayLikeSubscriber();
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub);
await driver.InitializeAsync("{}", CancellationToken.None);
var last = new System.Collections.Concurrent.ConcurrentDictionary<string, object?>();
driver.OnDataChange += (_, e) => last[e.FullReference] = e.Snapshot.Value;
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
await TestWait.UntilAsync(() => last.TryGetValue("Tank01.Level", out var v) && v is true,
"positive control: the subscribe's initial value arrives", () => Describe(driver, sub));
sub.ScanStateRunning = false; // the fake's next initial value
await driver.InvokeReplayForTestAsync(CancellationToken.None);
await TestWait.UntilAsync(() => last.TryGetValue("Tank01.Level", out var v) && v is false,
"the replay's re-advise must deliver its new initial value",
() => Describe(driver, sub) + $" last=[{string.Join(",", last.Select(kv => $"{kv.Key}={kv.Value}"))}] parked={driver.ParkedEarlyEventCountForTests} h2=[{string.Join(",", driver.SubscriptionsForTests.ResolveSubscribers(2).Select(b => b.FullReference))}] h1=[{string.Join(",", driver.SubscriptionsForTests.ResolveSubscribers(1).Select(b => b.FullReference))}]");
sub.DroppedInitialValues.ShouldBe(0);
}
/// <summary>
/// A probe sync on a driver whose pump was never started (no platform source, e.g. the
/// DiscoverAsync path) still starts it before advising — the watcher's pre-subscribe hook, not just