fix(galaxy): #865 review round 4 - replay fails on a dead stream, backoff-aware wait

- ReplayAsync fails when its restarted pump's stream has already ended, so
  the supervisor backs off and stays Degraded instead of going Healthy over
  a dead stream (whose next advise re-report restarted at attempt 1 with no
  backoff - a tight reconnect loop).
- ReconnectSupervisor.AttemptInProgress (reopen through replay); advises
  wait for a stale pump's replacement only while an attempt is running, not
  for a whole backoff (the state still reads Reopening/Replaying there).
- ReconnectSupervisor: _disposed is set under the state lock and re-checked
  inside ReportTransportFailure, so a late report cannot start a recovery
  loop that StopAsync already missed.
- EventPump.HasEnded is a flag set before StreamRequested is released, and
  a stream that ends/faults on its first MoveNext is marked ended first.
- SubscriptionRegistry fast path: a miss now falls through to the locked
  path (lookup + session + window check atomically) instead of a lock-free
  re-lookup.
- Tests: replay over an immediately-ending stream fails; AttemptInProgress
  across attempt/backoff; ReportTransportFailure after stop; the
  reopen-window test uses a generous wait bound.

Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
Joseph Doherty
2026-11-11 13:07:26 -05:00
parent 1c1705fa77
commit 8653960ea3
6 changed files with 156 additions and 22 deletions
@@ -51,6 +51,11 @@ public sealed class ProbeInitialValueTests
/// <summary>Initial values pushed while no stream was attached — lost, as on the real gateway.</summary>
public int DroppedInitialValues;
private int _streams;
/// <summary>Streams beyond this count end immediately after attaching (a gateway refusing/ending them).</summary>
public int StreamsThatStayOpen = int.MaxValue;
/// <summary>The ScanState value the next advise reports.</summary>
public volatile bool ScanStateRunning = true;
@@ -90,6 +95,11 @@ public sealed class ProbeInitialValueTests
Calls.Add("stream");
_attached = mine;
}
if (Interlocked.Increment(ref _streams) > StreamsThatStayOpen)
{
lock (_gate) { if (ReferenceEquals(_attached, mine)) _attached = null; }
yield break; // the gateway completed this stream at once
}
try
{
await foreach (var ev in mine.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
@@ -208,6 +218,9 @@ public sealed class ProbeInitialValueTests
platformSource: new FixedPlatformSource(Platforms))
{
RecreateOverrideForTests = _ => Task.CompletedTask,
// Generous: the window advise must still be waiting when the replay below starts the pump, even
// on a stalled agent; the wait ends the moment that pump exists.
StreamRequestWait = TimeSpan.FromSeconds(20),
};
await driver.InitializeAsync("{}", CancellationToken.None);
await TestWait.UntilAsync(
@@ -315,6 +328,27 @@ public sealed class ProbeInitialValueTests
sub.DroppedInitialValues.ShouldBe(0);
}
/// <summary>
/// A replay whose restarted pump's stream ends at once must FAIL (the supervisor then backs off and
/// stays Degraded) rather than succeed onto a dead stream — succeeding went Healthy, and the next
/// advise's re-report started a new cycle with no backoff: a tight reconnect loop. Positive control:
/// the first replay, whose stream stays open, succeeds.
/// </summary>
[Fact]
public async Task Replay_fails_when_the_restarted_stream_ends_at_once()
{
var sub = new GatewayLikeSubscriber { StreamsThatStayOpen = 2 };
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub);
await driver.InitializeAsync("{}", CancellationToken.None);
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None); // stream 1
await driver.InvokeReplayForTestAsync(CancellationToken.None); // stream 2 stays open — succeeds
await Should.ThrowAsync<InvalidOperationException>(
() => driver.InvokeReplayForTestAsync(CancellationToken.None), // stream 3 ends at once
"a replay must not report success over a dead event stream");
}
/// <summary>
/// A probe sync on a driver whose pump was never started (no platform source, e.g. the
/// DiscoverAsync path) still starts it before advising — the watcher's pre-subscribe hook, not just
@@ -15,6 +15,56 @@ public sealed class ReconnectSupervisorTests
new(InitialBackoffOverride: TimeSpan.FromMilliseconds(5),
MaxBackoffOverride: TimeSpan.FromMilliseconds(20));
/// <summary>
/// #865: AttemptInProgress is true while a reopen (through the replay) runs and FALSE during the backoff
/// between attempts — when the state still reads Reopening. Advises wait for a replay only while one is
/// actually running, not for a whole backoff.
/// </summary>
[Fact]
public async Task AttemptInProgress_is_true_during_an_attempt_and_false_during_the_backoff()
{
var reopenEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var releaseReopen = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var reopens = 0;
using var sup = new ReconnectSupervisor(
reopen: async _ =>
{
if (Interlocked.Increment(ref reopens) == 1)
{
reopenEntered.TrySetResult();
await releaseReopen.Task;
throw new IOException("gateway still down");
}
},
replay: _ => Task.CompletedTask,
backoffDelay: (_, _, _) => TimeSpan.FromMinutes(10)); // park in the backoff
sup.AttemptInProgress.ShouldBeFalse("Healthy: no attempt");
sup.ReportTransportFailure(new IOException("drop"));
await TestWait.ForAsync(reopenEntered.Task, "the first reopen must start");
sup.AttemptInProgress.ShouldBeTrue("a reopen is running");
releaseReopen.SetResult(); // the reopen fails -> backoff of 10 min
await TestWait.UntilAsync(() => !sup.AttemptInProgress, "the failed attempt must clear the flag");
sup.CurrentState.ShouldBe(ReconnectSupervisor.State.Reopening, "the backoff still reads Reopening — why the flag exists");
}
/// <summary>#865: a transport failure reported after StopAsync never starts a recovery loop (it throws ODE).</summary>
[Fact]
public async Task ReportTransportFailure_after_stop_never_starts_a_loop()
{
var reopens = 0;
var sup = new ReconnectSupervisor(
reopen: _ => { Interlocked.Increment(ref reopens); return Task.CompletedTask; },
replay: _ => Task.CompletedTask,
options: FastBackoff());
await sup.StopAsync();
Should.Throw<ObjectDisposedException>(() => sup.ReportTransportFailure(new IOException("late")));
sup.CurrentState.ShouldBe(ReconnectSupervisor.State.Healthy);
Volatile.Read(ref reopens).ShouldBe(0);
}
/// <summary>Verifies that the supervisor starts in a healthy state with no errors.</summary>
[Fact]
public void InitialState_IsHealthy()