fix(galaxy): #865 review round 5 - a failure during the replay fails the attempt
ReconnectSupervisor swallowed a transport failure reported while Replaying, so a restarted event stream refused/ended a round trip after the replay began still let the replay go Healthy over a dead stream (and the next advise's re-report restarted at attempt 1 with no backoff). A failure reported during Replaying is now remembered and turns that attempt's success into a retry with backoff. ReplayAsync's synchronous dead-pump throw yields to a teardown's cancellation. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
@@ -1065,6 +1065,10 @@ public sealed class GalaxyDriver
|
||||
lock (_pumpLock) replayPump = _eventPump;
|
||||
if (replayPump is { HasEnded: true })
|
||||
{
|
||||
// A teardown cancelling this replay can end the pump too: that is a clean cancellation.
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
// The synchronous case; a stream that ends a round trip later reports a transport failure
|
||||
// while Replaying, which the supervisor itself turns into a failed attempt.
|
||||
throw new InvalidOperationException(
|
||||
"The Galaxy event stream ended during the reconnect replay — retrying after backoff.");
|
||||
}
|
||||
|
||||
@@ -106,6 +106,9 @@ public sealed class ReconnectSupervisor : IDisposable
|
||||
|
||||
private bool _attemptInProgress;
|
||||
|
||||
// Set (under _stateLock) by a ReportTransportFailure that arrives while Replaying; cleared on entering Replaying.
|
||||
private bool _faultDuringReplay;
|
||||
|
||||
/// <summary>
|
||||
/// True while a reopen or replay is actually running — false between attempts (the backoff delay,
|
||||
/// during which the state still reads Reopening/Replaying) and while Healthy. #865: an advise waits
|
||||
@@ -151,7 +154,10 @@ public sealed class ReconnectSupervisor : IDisposable
|
||||
_lastError = cause.Message;
|
||||
if (_state != State.Healthy)
|
||||
{
|
||||
// Already recovering — nothing else to do.
|
||||
// Already recovering — nothing else to do, except: a failure DURING the replay means the
|
||||
// replay ran onto a transport that is already gone again (#865 — e.g. the restarted event
|
||||
// stream refused or ended); the replay's success must not then be reported as Healthy.
|
||||
if (_state == State.Replaying) _faultDuringReplay = true;
|
||||
_logger.LogDebug("Transport failure reported during {State}: {Message}", _state, cause.Message);
|
||||
return;
|
||||
}
|
||||
@@ -241,7 +247,11 @@ public sealed class ReconnectSupervisor : IDisposable
|
||||
}
|
||||
|
||||
// === Replaying phase ===
|
||||
lock (_stateLock) TransitionLocked(State.Replaying, _lastError);
|
||||
lock (_stateLock)
|
||||
{
|
||||
_faultDuringReplay = false;
|
||||
TransitionLocked(State.Replaying, _lastError);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
@@ -259,11 +269,24 @@ public sealed class ReconnectSupervisor : IDisposable
|
||||
Volatile.Write(ref _attemptInProgress, false);
|
||||
}
|
||||
|
||||
// === Done ===
|
||||
// === Done — unless the transport failed again while the replay ran ===
|
||||
bool faultedDuringReplay;
|
||||
lock (_stateLock)
|
||||
{
|
||||
_lastError = null;
|
||||
TransitionLocked(State.Healthy, null);
|
||||
faultedDuringReplay = _faultDuringReplay;
|
||||
if (!faultedDuringReplay)
|
||||
{
|
||||
_lastError = null;
|
||||
TransitionLocked(State.Healthy, null);
|
||||
}
|
||||
}
|
||||
if (faultedDuringReplay)
|
||||
{
|
||||
// Going Healthy here would hide a dead transport (the fault was swallowed above), and the
|
||||
// next report would restart at attempt 1 with no backoff. Retry this cycle with backoff.
|
||||
_logger.LogWarning(
|
||||
"Galaxy replay completed but the transport failed again while it ran (attempt {Attempt}) — retrying", attempt);
|
||||
continue;
|
||||
}
|
||||
_logger.LogInformation("Galaxy reconnect succeeded after {Attempt} attempt(s)", attempt);
|
||||
return;
|
||||
|
||||
Reference in New Issue
Block a user